diff --git a/app/cnn/MkCCNApp.kl b/app/cnn/MkCCNApp.kl new file mode 100644 index 0000000..07f30c6 --- /dev/null +++ b/app/cnn/MkCCNApp.kl @@ -0,0 +1,47 @@ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// + +require MLKL; + + +operator entry() { + + String path_config = "C:/Users/Julien/Documents/Dev/MLKL/app/cnn/cnn_config.mlkl"; + String path_image = 'C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/4.bmp'; + + // Load the configuration file and set the network layers + MkCNNConfig config; + MkCNNLayer layers[]; + if(!config.config(path_config, layers)) + return; + + // Create the network + MkCNNNetwork nn(config, layers); + + // Set the initiale learning rate + nn.optimizer().learningRate(nn.optimizer().learningRate()* sqrt(config.batch_size)); + nn.display(); + + // Declare the enumeration, here we can set the larning rate decay + // However, it should be set from the file, see Convnet for this + MkCNNEnumEpoch epoch_enum(); + + // Load the training data + MkCNNData data = LoadTrainingCIFAR(config, true, -1.0, -1.0); + //data.loadTrainingMNIST(config); + + // Train the network + // The network (the layers' weights) is tested and saved at each epch + nn.train(config, data, epoch_enum); + + + // Finally test it and save it + Float64 image_data[][] = LoadValidationCIFAR(config, true, -1.0, -1.0); + report(nn.predictRescale(image_data)); +} + diff --git a/app/cnn/cnn_config.mlkl b/app/cnn/cnn_config.mlkl new file mode 100644 index 0000000..8e673ad --- /dev/null +++ b/app/cnn/cnn_config.mlkl @@ -0,0 +1,65 @@ +### MLKL network configuration + + +############################################################################## +# +# 1. Network parameters +# tool* : "train" or "predict" +# gpu : GPU (0, 1) +# nbEpoch : Number of epochs +# batchSize : Batch size +# optimizer : Optimisation (0 : GD, 1 : GDLM) +# lossFunction : Loss funtion (0 : MSE, 1 : CE) +# +# 2. Layer definitions +# layersDefsPath : Path to the layers definition file +# layersParamsPath : Path to the layers parameters file +# +# 3. Images and labels +# trainImagesPath : Path of the train images, use for training +# trainLabelsPath : Path of the train labels, use for training +# testImagesPath* : Path of the test images, use for training +# testLabelsPath* : Path of the test labels, use for training +# +# 4. Network saving and loading +# save : Save the netwok +# autoSaving* : Automatically save the network at each epoch (0, 1) +# saveDirPath : Path of the network output folder +# saveFileName* : Name of the network output file +# loadFilePath* : Path of the network file to load +# +############################################################################## + + +# 1. Network parameters +tool=train +gpu=0 +nbEpoch=5 +batchSize=10 +optimizer=3 +lossFunction=0 + + +# 2. Layer definitions +layersDefsPath=C:/Users/Julien/Documents/Dev/MLKL/app/cnn/cnn_layers_def.mlkl +layersParamsPath=C:/Users/Julien/Documents/Dev/MLKL/app/cnn/cnn_layers_params.mlkl + + +# 3. Images and labels +# For CIFAR, the same file contains both the image and labels +trainImagesPath=C:/Users/Julien/Documents/Dev/MLKL/resources/cifar-10/data_batch_1.bin +testImagesPath=C:/Users/Julien/Documents/Dev/MLKL/resources/cifar-10/test_batch.bin + +# For MNIST +#trainImagesPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-images.idx3-ubyte +#testImagesPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-images.idx3-ubyte +#trainLabelsPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-labels.idx1-ubyte +#testLabelsPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-labels.idx1-ubyte + + +# 4. Network saving and loading +save=1 +autoSaving=1 +saveDirPath=C:/Users/Julien/Documents/Dev/MLKL/resources/ +saveFileName=cnn_network.mlkl +#loadFilePath=C:/Users/Julien/Documents/Dev/MLKL/resources/2015-07-04_18-25-59/res.mlkl diff --git a/app/cnn/cnn_layers_def.mlkl b/app/cnn/cnn_layers_def.mlkl new file mode 100644 index 0000000..ce8e43e --- /dev/null +++ b/app/cnn/cnn_layers_def.mlkl @@ -0,0 +1,75 @@ +### MLKL layers definition + + +############################################################################## +# +# [name] : Name of the layer +# type : Type (data, conv, pool, fc) +# neuron : Neuron (tanh, relu, sigm, id) +# filters : number of filters +# filterSize : Filter size +# inChannels : Number of channels inputs +# outChannels : Number of channels outputs +# initW : Initiliaze the weights with a normal dist of mean initW +# initB : Add bias initB +# +############################################################################## + +[conv1] +type=conv +neuron=relu +filters=32 +filterSize=5 +inChannels=3 +outChannels=6 +#initW=0.0 +#initB=0.0 + +[pool1] +type=pool +pool=avg +neuron=relu +filters=28 +inChannels=6 +poolingSize=2 +#initW=0.0 +#initB=0.0 + +[conv2] +type=conv +neuron=relu +filters=14 +filterSize=5 +inChannels=6 +outChannels=16 +#initW=0.0 +#initB=0.0 + +[pool2] +type=pool +pool=avg +neuron=relu +filters=10 +inChannels=16 +poolingSize=2 +#initW=0.0 +#initB=0.0 + +[conv3] +type=conv +neuron=relu +filters=5 +filterSize=5 +inChannels=16 +outChannels=120 +#initW=0.0 +#initB=0.0 + +[fc3] +type=fc +neuron=relu +inChannels=120 +outChannels=10 +#initW=0.0 +#initB=0.0 + diff --git a/app/cnn/cnn_layers_def_64.mlkl b/app/cnn/cnn_layers_def_64.mlkl new file mode 100644 index 0000000..aa8875c --- /dev/null +++ b/app/cnn/cnn_layers_def_64.mlkl @@ -0,0 +1,75 @@ +### MLKL layers definition + + +############################################################################## +# +# [name] : Name of the layer +# type : Type (data, conv, pool, fc) +# neuron : Neuron (tanh, relu, sigm, id) +# filters : number of filters +# filterSize : Filter size +# inChannels : Number of channels inputs +# outChannels : Number of channels outputs +# initW : Initiliaze the weights with a normal dist of mean initW +# initB : Add bias initB +# +############################################################################## + +[conv1] +type=conv +neuron=relu +filters=64 +filterSize=5 +inChannels=3 +outChannels=6 +#initW=0.0 +#initB=0.0 + +[pool1] +type=pool +pool=avg +neuron=relu +filters=60 +inChannels=6 +poolingSize=2 +#initW=0.0 +#initB=0.0 + +[conv2] +type=conv +neuron=relu +filters=30 +filterSize=5 +inChannels=6 +outChannels=16 +#initW=0.0 +#initB=0.0 + +[pool2] +type=pool +pool=avg +neuron=relu +filters=26 +inChannels=16 +poolingSize=2 +#initW=0.0 +#initB=0.0 + +[conv3] +type=conv +neuron=relu +filters=13 +filterSize=5 +inChannels=16 +outChannels=120 +#initW=0.0 +#initB=0.0 + +[fc3] +type=fc +neuron=relu +inChannels=9720 +outChannels=10 +#initW=0.0 +#initB=0.0 + diff --git a/app/cnn/cnn_layers_params.mlkl b/app/cnn/cnn_layers_params.mlkl new file mode 100644 index 0000000..3c42570 --- /dev/null +++ b/app/cnn/cnn_layers_params.mlkl @@ -0,0 +1,41 @@ +### MLKL layers parameters + + +############################################################################## +# +# [name] : Name of the layer +# epsW : Type (data, conv, pool, fc) +# epsB : Neuron (relu, relu, exp) +# momW : +# momB : Windowing size +# wc : Number of inputs +# +############################################################################## + +[conv1] +epsW=1 +epsB=1 +momW=1 +momB=1 +wc=1 + +[conv2] +epsW=2 +epsB=2 +momW=2 +momB=2 +wc=2 + +[conv3] +epsW=3 +epsB=3 +momW=3 +momB=3 +wc=3 + +[fc3] +epsW=4 +epsB=4 +momW=4 +momB=4 +wc=4 diff --git a/app/cnn/sample/MkCCNSample_DAE.kl b/app/cnn/sample/MkCCNSample_DAE.kl new file mode 100644 index 0000000..ef5961d --- /dev/null +++ b/app/cnn/sample/MkCCNSample_DAE.kl @@ -0,0 +1,59 @@ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// + +require MLKL; + +function MkCNNConfig ConfigMLP() { + MkCNNConfig config; + + config.save=0; + config.gpu=0; + config.epoch = 20; + config.batch_size = 10; + config.optimizer = MK_OPTIMIZER_GD; + config.loss_function = MK_LOSS_MSE; + + config.train_images_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-images.idx3-ubyte"; + config.test_images_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-images.idx3-ubyte"; + config.train_labels_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-labels.idx1-ubyte"; + config.test_labels_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-labels.idx1-ubyte"; + + return config; +} + +function MkCNNLayer[] ConstructMLP( + Index nb_input_units, + Index nb_hidden_units, + Index nb_output_units) +{ + MkCNNLayer layers[]; + //layers.push(MkCNNLayerConvolutional(MK_NEURON_TANH, 32, 5, 5, 16, 120)); + layers.push(MkCNNLayerFully(MK_NEURON_TANH, nb_input_units, nb_hidden_units)); + layers.push(MkCNNLayerFully(MK_NEURON_TANH, nb_hidden_units, nb_output_units)); + return layers; +} + +operator entry() { + + Index nb_hidden_neuron = 200; + MkCNNConfig config = ConfigMLP(); + MkCNNLayer layers[] = ConstructMLP(32*32, nb_hidden_neuron, 10); + MkCNNNetwork nn(config.loss_function, config.optimizer, layers); + + nn.display(); + // load MNIST dataset + MkCNNData data; + data.loadTrainingMNIST(config); + + nn.optimizer().learningRate(0.001); + + // Declare the enumeration, here we can set the larning rate decay + MkCNNEnumEpoch epoch_enum(); + + nn.train(data, config, epoch_enum); +} diff --git a/app/cnn/sample/MkCCNSample_Dropout.kl b/app/cnn/sample/MkCCNSample_Dropout.kl new file mode 100644 index 0000000..30d899d --- /dev/null +++ b/app/cnn/sample/MkCCNSample_Dropout.kl @@ -0,0 +1,59 @@ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// + +require MLKL; + +function MkCNNConfig ConfigDropout() { + MkCNNConfig config; + + config.save=0; + config.gpu=0; + config.epoch = 100; + config.batch_size = 1; + config.optimizer = MK_OPTIMIZER_GD; + config.loss_function = MK_LOSS_MSE; + + config.train_images_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-images.idx3-ubyte"; + config.test_images_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-images.idx3-ubyte"; + config.train_labels_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-labels.idx1-ubyte"; + config.test_labels_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-labels.idx1-ubyte"; + + return config; +} + +function MkCNNLayer[] ConstructDropout( + Index nb_input_units, + Index nb_hidden_units, + Index nb_output_units) +{ + MkCNNLayer layers[]; + //layers.push(MkCNNLayerConvolutional(MK_NEURON_TANH, 32, 5, 5, 16, 120)); + layers.push(MkCNNLayerDropout(MK_NEURON_TANH, nb_input_units, nb_hidden_units, DROPOUT_MODE_PER_DATA)); + layers.push(MkCNNLayerFully(MK_NEURON_TANH, nb_hidden_units, nb_output_units)); + return layers; +} + +operator entry() { + + Index nb_hidden_neuron = 200; + MkCNNConfig config = ConfigDropout(); + MkCNNLayer layers[] = ConstructDropout(32*32, nb_hidden_neuron, 10); + MkCNNNetwork nn(config.loss_function, config.optimizer, layers); + + nn.display(); + // load MNIST dataset + MkCNNData data; + data.loadTrainingMNIST(config); + + nn.optimizer().learningRate(0.003); + + // Declare the enumeration, here we can set the larning rate decay + MkCNNEnumEpoch epoch_enum(); + + nn.train(data, config, epoch_enum); +} diff --git a/app/tests/MkCNNUnitTest.kl b/app/cnn/sample/MkCCNSample_LeNet.kl similarity index 52% rename from app/tests/MkCNNUnitTest.kl rename to app/cnn/sample/MkCCNSample_LeNet.kl index bbc5f8f..eacb4bb 100644 --- a/app/tests/MkCNNUnitTest.kl +++ b/app/cnn/sample/MkCCNSample_LeNet.kl @@ -1,16 +1,34 @@ -/**************************************************************************************************/ -/* */ -/* Informations : */ -/* This code is part of the project MLKL */ -/* */ -/* Contacts : */ -/* couet.julien@gmail.com */ -/* */ -/**************************************************************************************************/ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// require MLKL; + +function MkCNNConfig ConfigLeNet() { + MkCNNConfig config; + + config.save=0; + config.gpu=0; + config.epoch = 20; + config.batch_size = 10; + config.optimizer = MK_OPTIMIZER_GD; + config.loss_function = MK_LOSS_MSE; + + config.train_images_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-images.idx3-ubyte"; + config.test_images_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-images.idx3-ubyte"; + config.train_labels_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-labels.idx1-ubyte"; + config.test_labels_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-labels.idx1-ubyte"; + + return config; +} + function Boolean[] CreateConnections() { + Boolean O = true; Boolean X = false; @@ -30,8 +48,8 @@ function Boolean[] CreateConnections() { return c; } -function MkCNNLayerInterface[] CreateLayers() { - MkCNNLayerInterface layers[]; +function MkCNNLayer[] ConstructLeNet() { + MkCNNLayer layers[]; layers.push(MkCNNLayerConvolutional(MK_NEURON_TANH, 32, 32, 5, 1, 6)); layers.push(MkCNNLayerAveragePooling(MK_NEURON_TANH, 28, 28, 6, 2)); layers.push(MkCNNLayerConvolutional(MK_NEURON_TANH, 14, 14, 5, 6, 16, MkCNNConnectionTable(CreateConnections(), 6, 16))); @@ -41,50 +59,31 @@ function MkCNNLayerInterface[] CreateLayers() { return layers; } -function LoadMNIST( - MkCNNConfig config, - io Float64 train_images[][], - io Float64 test_images[][], - io Index train_labels[], - io Index test_labels[]) -{ - MkMNIST mnist(); - train_images = mnist.parseImages(config.train_images_path, -1.0, 1.0, 2, 2); - test_images = mnist.parseImages(config.test_images_path, -1.0, 1.0, 2, 2); - train_labels = mnist.parseLabels(config.train_labels_path); - test_labels = mnist.parseLabels(config.test_labels_path); -} - operator entry() { - // Load the data - Index train_labels[], test_labels[]; - Float64 train_images[][], test_images[][]; - LoadMNIST(config, train_images, test_images, train_labels, test_labels); - // Configure the network layers - MkCNNConfig config; - MkCNNLayerInterface layers = CreateLayers(); + MkCNNConfig config = ConfigLeNet(); + MkCNNLayer layers = ConstructLeNet(); // Create the network MkCNNNetwork nn(config.lossFunction(), config.optimizer(), layers); - + nn.display(); + // Set the initiale learning rate Ref opti = nn.optimizer(); nn.optimizer().learningRate(nn.optimizer().learningRate()* sqrt(config.batch_size)); - // Declare the enumeration + // load MNIST dataset + MkCNNData data; + data.loadTrainingMNIST(config); + + // Train the network // Here we could sate the larning rate decay MkEnumerateEpoch on_epoch_enumerate(); - MkEnumerateData on_batch_enumerate(train_images.size(), config.batch_size); - - // Train the network - nn.train( config, train_images, train_labels, test_images, test_labels, - on_epoch_enumerate, on_batch_enumerate); + nn.train(data, config, epoch_enum); // Finally test it - String path_image = 'C:/Users/Julien/Documents/Dev/MLKL/data/mnist/4.bmp'; - Float64 image_data[] = LoadImage(path_image, -1.0, 1.0); + //String path_image = 'C:/Users/Julien/Documents/Dev/MLKL/data/mnist/4.bmp'; + //Float64 image_data[] = LoadImage(path_image, -1.0, 1.0); } - diff --git a/app/cnn/sample/MkCCNSample_MLP.kl b/app/cnn/sample/MkCCNSample_MLP.kl new file mode 100644 index 0000000..b928bf0 --- /dev/null +++ b/app/cnn/sample/MkCCNSample_MLP.kl @@ -0,0 +1,58 @@ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// + +require MLKL; + +function MkCNNConfig ConfigMLP() { + MkCNNConfig config; + + config.save=0; + config.gpu=0; + config.epoch = 20; + config.batch_size = 1; + config.optimizer = MK_OPTIMIZER_GD; + config.loss_function = MK_LOSS_MSE; + + config.train_images_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-images.idx3-ubyte"; + config.train_labels_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-labels.idx1-ubyte"; + config.test_images_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-images.idx3-ubyte"; + config.test_labels_path = "C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-labels.idx1-ubyte"; + + return config; +} + +function MkCNNLayer[] ConstructMLP( + Index nb_input_units, + Index nb_hidden_units, + Index nb_output_units) +{ + MkCNNLayer layers[]; + layers.push(MkCNNLayerFully(MK_NEURON_TANH, nb_input_units, nb_hidden_units)); + layers.push(MkCNNLayerFully(MK_NEURON_TANH, nb_hidden_units, nb_output_units)); + return layers; +} + +operator entry() { + + Index nb_hidden_neuron = 200; + MkCNNConfig config = ConfigMLP(); + MkCNNLayer layers[] = ConstructMLP(32*32, nb_hidden_neuron, 10); + MkCNNNetwork nn(config.loss_function, config.optimizer, layers); + nn.display(); + + // load MNIST dataset + MkCNNData data; + data.loadTrainingMNIST(config); + + nn.optimizer().learningRate(0.001); + + // Declare the enumeration, here we can set the larning rate decay + MkCNNEnumEpoch epoch_enum(); + + nn.train(data, config, epoch_enum); +} diff --git a/app/cnn/test/MkCNNConvFPropTest.kl b/app/cnn/test/MkCNNConvFPropTest.kl new file mode 100644 index 0000000..e69de29 diff --git a/app/cnn/test/MkCNNConvGradCheckTest1.kl b/app/cnn/test/MkCNNConvGradCheckTest1.kl new file mode 100644 index 0000000..e69de29 diff --git a/app/cnn/test/MkCNNConvGradCheckTest2.kl b/app/cnn/test/MkCNNConvGradCheckTest2.kl new file mode 100644 index 0000000..e69de29 diff --git a/app/cnn/test/MkCNNConvGradCheckTest3.kl b/app/cnn/test/MkCNNConvGradCheckTest3.kl new file mode 100644 index 0000000..e69de29 diff --git a/app/cnn/test/MkCNNConvGradCheckTest4.kl b/app/cnn/test/MkCNNConvGradCheckTest4.kl new file mode 100644 index 0000000..e69de29 diff --git a/app/cnn/test/MkCNNConvGradCheckTest5.kl b/app/cnn/test/MkCNNConvGradCheckTest5.kl new file mode 100644 index 0000000..e69de29 diff --git a/app/cnn/test/MkCNNConvSerializeTest1.kl b/app/cnn/test/MkCNNConvSerializeTest1.kl new file mode 100644 index 0000000..e69de29 diff --git a/app/cnn/test/MkCNNConvSerializeTest2.kl b/app/cnn/test/MkCNNConvSerializeTest2.kl new file mode 100644 index 0000000..e69de29 diff --git a/config/environment.bat b/app/config.bat similarity index 52% rename from config/environment.bat rename to app/config.bat index b8faa48..d88534c 100644 --- a/config/environment.bat +++ b/app/config.bat @@ -8,10 +8,9 @@ set PYTHON_VERSION=2.7 set PYTHONPATH=%FABRIC_DIR%\Python\%PYTHON_VERSION%;%PYTHONPATH% set PYTHON_VERSION=%_OLD_PYTHON_VERSION% -set MLKL_MNIST_EXT=C:\Users\Julien\Documents\Dev\MLKL\core\c++\mnist -set MLKL_CIFAR_EXT=C:\Users\Julien\Documents\Dev\MLKL\core\c++\cifar -set MLKL_EXTS=C:\Users\Julien\Documents\Dev\MLKL\core\kl +set MLKL_MNIST_EXT=C:\Users\Julien\Documents\Dev\MLKL\core\cnn\c++ +set MLKL_EXTS=C:\Users\Julien\Documents\Dev\MLKL\core set FABRIC_EXTS_PATH=%FABRIC_DIR%\Exts -set FABRIC_EXTS_PATH=%MLKL_CIFAR_EXT%;%MLKL_MNIST_EXT%;%MLKL_EXTS%;%FABRIC_EXTS_PATH% +set FABRIC_EXTS_PATH=%MLKL_MNIST_EXT%;%MLKL_EXTS%;%FABRIC_EXTS_PATH% diff --git a/config/environment.sh b/app/config.sh similarity index 100% rename from config/environment.sh rename to app/config.sh diff --git a/app/samples/cnn/MkCCNApp.kl b/app/samples/cnn/MkCCNApp.kl deleted file mode 100644 index 431254f..0000000 --- a/app/samples/cnn/MkCCNApp.kl +++ /dev/null @@ -1,55 +0,0 @@ -/**************************************************************************************************/ -/* */ -/* Informations : */ -/* This code is part of the project MLKL */ -/* */ -/* Contacts : */ -/* couet.julien@gmail.com */ -/* */ -/**************************************************************************************************/ - -require MLKL; - - -operator entry() { - - String path_config = "C:/Users/Julien/Documents/Dev/MLKL/app/samples/cnn/cnn_config.mlkl"; - String path_image = 'C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/4.bmp'; - - // Load the configuration file and set the network layers - MkCNNConfig config; - MkCNNLayerInterface layers[]; - if(!config.config(path_config, layers)) - return; - - // Create the network - MkCNNNetwork nn(config.lossFunction(), config.optimizer(), layers); - - // Set the initiale learning rate - Ref opti = nn.optimizer(); - nn.optimizer().learningRate(nn.optimizer().learningRate()* sqrt(config.batch_size)); - - // Declare the enumeration, here we can set the larning rate decay - // However, it should be set from the file, see Convnet for this - MkEnumerateEpoch on_epoch_enumerate(); - - // Load the training data - //MkCNNTrainingData training_data = LoadTrainingData_MNIST(config); - MkCNNTrainingData training_data = LoadTrainingData_CIFAR(config); - report("train_images.size " + training_data.train_images.size()); - report("train_labels.size " + training_data.train_labels.size()); - report("tests_images.size " + training_data.test_images.size()); - report("tests_labels.size " + training_data.test_labels.size()); - - report("training_data.train_images[0] " + training_data.train_images[0]); - - // Train the network - // The network (the layers' weights) is tested and saved at each epch - nn.train(training_data, config, on_epoch_enumerate); - - - // Finally test it and save it - //Float64 image_data[] = LoadValidationData_MNIST(path_image, -1.0, 1.0); - //report(nn.predictRescale(image_data)); -} - diff --git a/app/samples/cnn/cnn_config.mlkl b/app/samples/cnn/cnn_config.mlkl deleted file mode 100644 index 413c162..0000000 --- a/app/samples/cnn/cnn_config.mlkl +++ /dev/null @@ -1,35 +0,0 @@ -# MLKL Network configuration file -# this is comment - -# Network configuration -# auto_saving=1 # to save at each epoch -# tool=train #to train the netweork, tool=predict to get output from network - -worker=1 -gpu=0 -nbEpoch=2 -batchSize=100 -optimizer=1 -lossFunction=0 - -# Set the layer def et params pathes -layersDefsPath=C:/Users/Julien/Documents/Dev/MLKL/app/samples/cnn/cnn_layers_def.mlkl -layersParamsPath=C:/Users/Julien/Documents/Dev/MLKL/app/samples/cnn/cnn_layers_params.mlkl - - -# For CIFAR -trainImagesPath=C:/Users/Julien/Documents/Dev/MLKL/resources/cifar-10/data_batch_1.bin -testImagesPath=C:/Users/Julien/Documents/Dev/MLKL/resources/cifar-10/test_batch.bin -trainLabelsPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-labels.idx1-ubyte -testLabelsPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-labels.idx1-ubyte - -# Set the output -outputDirPath=C:/Users/Julien/Documents/Dev/MLKL/resources/ - -# Set data pathes -# For MNIST - -#trainImagesPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-images.idx3-ubyte -#testImagesPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-images.idx3-ubyte -#trainLabelsPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/train-labels.idx1-ubyte -#testLabelsPath=C:/Users/Julien/Documents/Dev/MLKL/resources/mnist/t10k-labels.idx1-ubyte diff --git a/app/samples/cnn/cnn_layers_def.mlkl b/app/samples/cnn/cnn_layers_def.mlkl deleted file mode 100644 index 335d6a1..0000000 --- a/app/samples/cnn/cnn_layers_def.mlkl +++ /dev/null @@ -1,71 +0,0 @@ -# MLKL Network layers definition file - -# How to declare a layer -# [conv1] : Name of the layer -# type=conv : Type (data, conv, pool, fc) -# neuron=relu : Neuron (relu, relu, exp) -# filters=32 : -# filterSize=5 : Windowing size -# inChannels=1 : Number of inputs -# outChannels=6 : Number of outputs -# initW=0.1 : Initiliaze the layers' weights with a normal distribution of std initW -# initB=0.5 : Initiliaze the layers' weights with a normal distribution of std initB - -[conv1] -type=conv -neuron=relu -filters=32 -filterSize=5 -inChannels=1 -outChannels=6 -initW=0.0001 -initB=0.0 - -[pool1] -type=pool -pool=max -neuron=relu -filters=28 -inChannels=6 -poolingSize=2 -initW=0.0 -initB=0.0 - -[conv2] -type=conv -neuron=relu -filters=14 -filterSize=5 -inChannels=6 -outChannels=16 -initW=0.1 -initB=0.0 - -[pool2] -type=pool -pool=max -neuron=relu -filters=10 -inChannels=16 -poolingSize=2 -initW=0.0 -initB=0.0 - -[conv3] -type=conv -neuron=relu -filters=5 -filterSize=5 -inChannels=16 -outChannels=120 -initW=0.1 -initB=0.0 - -[fc3] -type=fc -neuron=relu -inChannels=120 -outChannels=10 -initW=0.0 -initB=0.0 - diff --git a/app/samples/cnn/cnn_layers_params.mlkl b/app/samples/cnn/cnn_layers_params.mlkl deleted file mode 100644 index 6953542..0000000 --- a/app/samples/cnn/cnn_layers_params.mlkl +++ /dev/null @@ -1,36 +0,0 @@ -# MLKL Network layers parameters file - -#[conv2] -#epsW=0.001 -#epsB=0.002 -#momW=0.9 -#momB=0.9 -#wc=0.000 - -[conv1] -epsW=1 -epsB=1 -momW=1 -momB=1 -wc=1 - -[conv2] -epsW=2 -epsB=2 -momW=2 -momB=2 -wc=2 - -[conv3] -epsW=3 -epsB=3 -momW=3 -momB=3 -wc=3 - -[fc3] -epsW=4 -epsB=4 -momW=4 -momB=4 -wc=4 diff --git a/app/samples/MkSVMApp.kl b/app/svm/MkSVMApp.kl similarity index 100% rename from app/samples/MkSVMApp.kl rename to app/svm/MkSVMApp.kl diff --git a/core/kl/MLKL.fpm.json b/core/MLKL.fpm.json similarity index 100% rename from core/kl/MLKL.fpm.json rename to core/MLKL.fpm.json diff --git a/core/c++/MNIST/.sconsign.dblite b/core/c++/MNIST/.sconsign.dblite deleted file mode 100644 index 269c046..0000000 Binary files a/core/c++/MNIST/.sconsign.dblite and /dev/null differ diff --git a/core/c++/MNIST/MkMNIST-Windows-x86_64.exp b/core/c++/MNIST/MkMNIST-Windows-x86_64.exp deleted file mode 100644 index 19e7cd4..0000000 Binary files a/core/c++/MNIST/MkMNIST-Windows-x86_64.exp and /dev/null differ diff --git a/core/c++/MNIST/MkMNIST.fpm.json b/core/c++/MNIST/MkMNIST.fpm.json deleted file mode 100644 index 5e1155b..0000000 --- a/core/c++/MNIST/MkMNIST.fpm.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "libs": "MkMNIST", - "code": ["MkMNIST.kl" ] -} diff --git a/core/c++/MNIST/MkMNIST.h b/core/c++/MNIST/MkMNIST.h deleted file mode 100644 index a52271d..0000000 --- a/core/c++/MNIST/MkMNIST.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_MkMNIST__ -#define __KL2EDK_AUTOGEN_MkMNIST__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'MkMNIST.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" - -#include "MkMNIST_functions.h" -#include "MkMNIST_impl.h" -#include "MkMNIST_type.h" -#include "global_functions.h" - -#endif // __KL2EDK_AUTOGEN_MkMNIST__ diff --git a/core/c++/MNIST/MkMNIST_functions.h b/core/c++/MNIST/MkMNIST_functions.h deleted file mode 100644 index 3b7e0e5..0000000 --- a/core/c++/MNIST/MkMNIST_functions.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_MkMNIST_functions__ -#define __KL2EDK_AUTOGEN_MkMNIST_functions__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'MkMNIST_functions.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" -#include "MkMNIST_type.h" - - -// Defined at MkMNIST.kl:15:1 -FABRIC_EXT_EXPORT void MkMNIST_parseLabels( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::UInt32 > >::Result _result, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::MkMNIST >::INParam this_, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::String >::INParam path -); - -// Defined at MkMNIST.kl:17:1 -FABRIC_EXT_EXPORT void MkMNIST_parseImages( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::Float64 > > >::Result _result, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::MkMNIST >::INParam this_, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::String >::INParam path, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam scale_min, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam scale_max, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam x_padding, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam y_padding -); - -#endif // __KL2EDK_AUTOGEN_MkMNIST_functions__ diff --git a/core/c++/MNIST/MkMNIST_impl.h b/core/c++/MNIST/MkMNIST_impl.h deleted file mode 100644 index d221a6a..0000000 --- a/core/c++/MNIST/MkMNIST_impl.h +++ /dev/null @@ -1,181 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_MkMNIST_impl__ -#define __KL2EDK_AUTOGEN_MkMNIST_impl__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'MkMNIST_impl.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" - -#include "MkMNIST.h" - -namespace Fabric { namespace EDK { namespace KL { - - struct MkMNIST::Bits - { - ObjectCore __objectCore; - Data handle; - Object::Bits __interfaceObjectBits; - }; - - inline void MkMNIST::ConstructEmpty( MkMNIST *self ) - { - self->m_bits = 0; - } - - inline void MkMNIST::ConstructCopy( MkMNIST *self, MkMNIST const *other ) - { - if ( (self->m_bits = other->m_bits) ) - AtomicUInt32Increment( &self->m_bits->__objectCore.refCount ); - } - - inline void MkMNIST::AssignCopy( MkMNIST *self, MkMNIST const *other ) - { - if ( self->m_bits != other->m_bits ) - { - Destruct( self ); - ConstructCopy( self, other ); - } - } - - inline void MkMNIST::Destruct( MkMNIST *self ) - { - if ( self->m_bits - && AtomicUInt32DecrementAndGetValue( &self->m_bits->__objectCore.refCount ) == 0 ) - { - ObjectCore *objectCorePtr = &self->m_bits->__objectCore; - self->m_bits->__objectCore.lTableSwapPtrPtr->get()->lifecycleDestroy( - &objectCorePtr - ); - } - } - - inline MkMNIST::MkMNIST() - { - ConstructEmpty( this ); - } - - inline MkMNIST MkMNIST::Create() - { - static KL::SwapPtr const *sp = 0; - if ( !sp ) - { - sp = static_cast const *>( - s_callbacks.m_lookupGlobalSymbol( - "sp.function.kl.OO_MkMNIST.createEmpty.io_AS0.OO_MkMNIST", - 55 - ) - ); - if ( !sp ) - throwException( - "EDK internal error: failed to look up '%s'", - "sp.function.kl.OO_MkMNIST.createEmpty.io_AS0.OO_MkMNIST" - ); - } - void (*createFuncPtr)(void *) = ((void (*)(void *))sp->get()); - if ( !createFuncPtr ) - throwException( - "EDK internal error: target of '%s' is NULL", - "sp.function.kl.OO_MkMNIST.createEmpty.io_AS0.OO_MkMNIST" - ); - MkMNIST result; - createFuncPtr( &result ); - return result; - } - - inline MkMNIST::MkMNIST( MkMNIST const &that ) - { - ConstructCopy( this, &that ); - } - - inline MkMNIST &MkMNIST::operator =( MkMNIST const &that ) - { - AssignCopy( this, &that ); - return *this; - } - - inline MkMNIST::~MkMNIST() - { - Destruct( this ); - } - - inline void MkMNIST::appendDesc( String::IOParam string ) const - { - if ( m_bits ) - { - ObjectCore *objectCorePtr = &m_bits->__objectCore; - objectCorePtr->lTableSwapPtrPtr->get()->appendDesc( - &objectCorePtr, string - ); - } - else string.append( "null", 4 ); - } - inline uint32_t MkMNIST::getRefCount() const - { - if ( m_bits ) - { - ObjectCore *objectCorePtr = &m_bits->__objectCore; - return objectCorePtr->refCount; - } - else return 0; - } - inline Type MkMNIST::getType() const - { - if ( m_bits ) - { - return Type( m_bits->__objectCore.typeInfoSwapPtrPtr->get() ); - } - else return Type(); - } - - inline bool MkMNIST::isValid() const - { - return !!m_bits; - } - - inline MkMNIST::operator bool() const - { - return isValid(); - } - - inline bool MkMNIST::operator !() const - { - return !isValid(); - } - - inline MkMNIST::Bits *MkMNIST::operator ->() - { - return static_cast( m_bits ); - } - - inline MkMNIST::Bits const *MkMNIST::operator ->() const - { - return static_cast( m_bits ); - } - - inline bool MkMNIST::operator ==( INParam that ) - { - return m_bits == that.m_bits; - } - - inline bool MkMNIST::operator !=( INParam that ) - { - return m_bits != that.m_bits; - } - - // Methods for 'Object' - -}}} - -#endif // __KL2EDK_AUTOGEN_MkMNIST_impl__ diff --git a/core/c++/MNIST/MkMNIST_type.h b/core/c++/MNIST/MkMNIST_type.h deleted file mode 100644 index f689112..0000000 --- a/core/c++/MNIST/MkMNIST_type.h +++ /dev/null @@ -1,99 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_MkMNIST_type__ -#define __KL2EDK_AUTOGEN_MkMNIST_type__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'MkMNIST_type.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" - -namespace Fabric { namespace EDK { namespace KL { - -// KL object 'MkMNIST' -// Defined at MkMNIST.kl:11:1 - -class MkMNIST -{ -protected: - - struct Bits; - Bits *m_bits; - - friend struct Traits< MkMNIST >; - - static void ConstructEmpty( MkMNIST *self ); - - static void ConstructCopy( MkMNIST *self, MkMNIST const *other ); - - static void AssignCopy( MkMNIST *self, MkMNIST const *other ); - - static void Destruct( MkMNIST *self ); - -public: - - typedef MkMNIST &Result; - typedef MkMNIST const &INParam; - typedef MkMNIST &IOParam; - - MkMNIST(); - - static MkMNIST Create(); - - MkMNIST( MkMNIST const &that ); - - MkMNIST &operator =( MkMNIST const &that ); - - ~MkMNIST(); - - void appendDesc( String::IOParam string ) const; - uint32_t getRefCount() const; - Type getType() const; - - bool isValid() const; - - operator bool() const; - - bool operator !() const; - - Bits *operator ->(); - - Bits const *operator ->() const; - - bool operator ==( INParam that ); - - bool operator !=( INParam that ); - - // Methods for 'Object' -}; - -inline void Traits::ConstructEmpty( MkMNIST &val ) -{ - MkMNIST::ConstructEmpty( &val ); -} -inline void Traits::ConstructCopy( MkMNIST &lhs, MkMNIST const &rhs ) -{ - MkMNIST::ConstructCopy( &lhs, &rhs ); -} -inline void Traits::AssignCopy( MkMNIST &lhs, MkMNIST const &rhs ) -{ - MkMNIST::AssignCopy( &lhs, &rhs ); -} -inline void Traits::Destruct( MkMNIST &val ) -{ - MkMNIST::Destruct( &val ); -} - -}}} - -#endif // __KL2EDK_AUTOGEN_MkMNIST_type__ diff --git a/core/c++/MNIST/aliases.h b/core/c++/MNIST/aliases.h deleted file mode 100644 index ee3864c..0000000 --- a/core/c++/MNIST/aliases.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_aliases__ -#define __KL2EDK_AUTOGEN_aliases__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'aliases.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -namespace Fabric { namespace EDK { namespace KL { - -}}} - -#endif // __KL2EDK_AUTOGEN_aliases__ diff --git a/core/c++/MNIST/global.h b/core/c++/MNIST/global.h deleted file mode 100644 index f5872c3..0000000 --- a/core/c++/MNIST/global.h +++ /dev/null @@ -1,193 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_global__ -#define __KL2EDK_AUTOGEN_global__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'global.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -// Dependencies on other extensions -#define FABRIC_EDK_EXT_MkMNIST_DEPENDENT_EXTS \ - { \ - { 0, 0, 0, 0, 0 } \ - } - -// forward declarations -namespace Fabric { namespace EDK { namespace KL { - class Object; - class MkMNIST; -}}} - -#include "aliases.h" - -namespace Fabric { namespace EDK { namespace KL { - -// KL interface 'Object' -// Defined at (internal) - -class Object -{ -public: - - struct VTable - { - }; - - struct Bits - { - ObjectCore *objectCorePtr; - SwapPtr const *vTableSwapPtrPtr; - } *m_bits; - -protected: - - friend struct Traits< Object >; - - static void ConstructEmpty( Object *self ) - { - self->m_bits = 0; - } - - static void ConstructCopy( Object *self, Object const *other ) - { - if ( (self->m_bits = other->m_bits) ) - AtomicUInt32Increment( &self->m_bits->objectCorePtr->refCount ); - } - - static void AssignCopy( Object *self, Object const *other ) - { - if ( self->m_bits != other->m_bits ) - { - Destruct( self ); - ConstructCopy( self, other ); - } - } - - static void Destruct( Object *self ) - { - if ( self->m_bits - && AtomicUInt32DecrementAndGetValue( &self->m_bits->objectCorePtr->refCount ) == 0 ) - { - self->m_bits->objectCorePtr->lTableSwapPtrPtr->get()->lifecycleDestroy( - &self->m_bits->objectCorePtr - ); - } - } - -public: - - typedef Object &Result; - typedef Object const &INParam; - typedef Object &IOParam; - - Object() - { - ConstructEmpty( this ); - } - - Object( Object const &that ) - { - ConstructCopy( this, &that ); - } - - Object &operator =( Object const &that ) - { - AssignCopy( this, &that ); - return *this; - } - - ~Object() - { - Destruct( this ); - } - - void appendDesc( String::IOParam string ) const - { - if ( m_bits ) - m_bits->objectCorePtr->lTableSwapPtrPtr->get()->appendDesc( &m_bits->objectCorePtr, string ); - else string.append( "null", 4 ); - } - - bool isValid() const - { - return !!m_bits; - } - - operator bool() const - { - return isValid(); - } - - bool operator !() const - { - return !isValid(); - } - - bool operator ==( INParam that ) - { - return m_bits == that.m_bits; - } - - bool operator !=( INParam that ) - { - return m_bits != that.m_bits; - } - -}; - -template<> -struct Traits< Object > -{ - typedef Object &Result; - typedef Object const &INParam; - typedef Object &IOParam; - - static void ConstructEmpty( Object &val ); - static void ConstructCopy( Object &lhs, Object const &rhs ); - static void AssignCopy( Object &lhs, Object const &rhs ); - static void Destruct( Object &val ); -}; - -inline void Traits::ConstructEmpty( Object &val ) -{ - Object::ConstructEmpty( &val ); -} -inline void Traits::ConstructCopy( Object &lhs, Object const &rhs ) -{ - Object::ConstructCopy( &lhs, &rhs ); -} -inline void Traits::AssignCopy( Object &lhs, Object const &rhs ) -{ - Object::AssignCopy( &lhs, &rhs ); -} -inline void Traits::Destruct( Object &val ) -{ - Object::Destruct( &val ); -} - -template<> -struct Traits< MkMNIST > -{ - typedef MkMNIST &Result; - typedef MkMNIST const &INParam; - typedef MkMNIST &IOParam; - - static void ConstructEmpty( MkMNIST &val ); - static void ConstructCopy( MkMNIST &lhs, MkMNIST const &rhs ); - static void AssignCopy( MkMNIST &lhs, MkMNIST const &rhs ); - static void Destruct( MkMNIST &val ); -}; - -}}} - -#endif // __KL2EDK_AUTOGEN_global__ diff --git a/core/c++/MNIST/global_functions.h b/core/c++/MNIST/global_functions.h deleted file mode 100644 index 6129a0d..0000000 --- a/core/c++/MNIST/global_functions.h +++ /dev/null @@ -1,112 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_global_functions__ -#define __KL2EDK_AUTOGEN_global_functions__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'global_functions.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" - - -// Defined at MkMNIST.kl:27:1 -FABRIC_EXT_EXPORT void UniformRand_Float64( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam min, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam max, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::IOParam res -); - -// Defined at MkMNIST.kl:29:1 -FABRIC_EXT_EXPORT void UniformRand_Float32( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float32 >::INParam min, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float32 >::INParam max, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float32 >::IOParam res -); - -// Defined at MkMNIST.kl:31:1 -FABRIC_EXT_EXPORT void UniformRand_SInt32( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::SInt32 >::INParam min, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::SInt32 >::INParam max, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::SInt32 >::IOParam res -); - -// Defined at MkMNIST.kl:33:1 -FABRIC_EXT_EXPORT void UniformRand_UInt32( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::UInt32 >::INParam min, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::UInt32 >::INParam max, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::UInt32 >::IOParam res -); - -// Defined at MkMNIST.kl:36:1 -FABRIC_EXT_EXPORT void UniformRealDistribution_Float64( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam min, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam max, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::Float64 > >::IOParam dist -); - -// Defined at MkMNIST.kl:38:1 -FABRIC_EXT_EXPORT void UniformRealDistribution_Float32( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float32 >::INParam min, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float32 >::INParam max, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::Float32 > >::IOParam dist -); - -// Defined at MkMNIST.kl:40:1 -FABRIC_EXT_EXPORT void UniformRealDistribution_UInt32( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::UInt32 >::INParam min, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::UInt32 >::INParam max, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::UInt32 > >::IOParam dist -); - -// Defined at MkMNIST.kl:42:1 -FABRIC_EXT_EXPORT void UniformRealDistribution_SInt32( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::SInt32 >::INParam min, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::SInt32 >::INParam max, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::SInt32 > >::IOParam dist -); - -// Defined at MkMNIST.kl:44:1 -FABRIC_EXT_EXPORT void Bernoulli_Float64( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam p, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Boolean >::IOParam res -); - -// Defined at MkMNIST.kl:46:1 -FABRIC_EXT_EXPORT void Bernoulli_Float32( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float32 >::INParam p, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Boolean >::IOParam res -); - -// Defined at MkMNIST.kl:48:1 -FABRIC_EXT_EXPORT void Bernoulli_Float64_UInt32( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float64 >::INParam p, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::UInt32 >::IOParam res -); - -// Defined at MkMNIST.kl:50:1 -FABRIC_EXT_EXPORT void Bernoulli_Float32_UInt32( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::Float32 >::INParam p, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::UInt32 >::IOParam res -); - -// Defined at MkMNIST.kl:53:1 -FABRIC_EXT_EXPORT void ReportR( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::String >::INParam str -); - -// Defined at MkMNIST.kl:55:1 -FABRIC_EXT_EXPORT void CurrentDateTime( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::String >::IOParam str -); - -#endif // __KL2EDK_AUTOGEN_global_functions__ diff --git a/core/c++/cifar/.sconsign.dblite b/core/c++/cifar/.sconsign.dblite deleted file mode 100644 index 560900f..0000000 Binary files a/core/c++/cifar/.sconsign.dblite and /dev/null differ diff --git a/core/c++/cifar/MkCIFAR-Windows-x86_64.exp b/core/c++/cifar/MkCIFAR-Windows-x86_64.exp deleted file mode 100644 index 186dd8e..0000000 Binary files a/core/c++/cifar/MkCIFAR-Windows-x86_64.exp and /dev/null differ diff --git a/core/c++/cifar/MkCIFAR.cpp b/core/c++/cifar/MkCIFAR.cpp deleted file mode 100644 index 46e45ed..0000000 --- a/core/c++/cifar/MkCIFAR.cpp +++ /dev/null @@ -1,247 +0,0 @@ -/**************************************************************************************************/ -/* */ -/* Informations : */ -/* This code is part of the project MLKL */ -/* */ -/* Contacts : */ -/* couet.julien@gmail.com */ -/* */ -/**************************************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include -#include -using namespace Fabric::EDK; - -IMPLEMENT_FABRIC_EDK_ENTRIES( MkCIFAR ) - - -void ReadBatch( - string filename, - KL::VariableArray > &images, - KL::VariableArray &labels) -{ - ifstream file (filename, ios::binary); - if (file.is_open()) - { - int number_of_images = 10000; - int n_rows = 32; - int n_cols = 32; - - double scale_max = 1.0; - double scale_min = -1.0; - - images.resize(number_of_images); - labels.resize(number_of_images); - - for(int i = 0; i < number_of_images; ++i) - { - unsigned char tplabel = 0; - file.read((char*) &tplabel, sizeof(tplabel)); - labels[i] = KL::UInt32(tplabel); - - KL::VariableArray > channels; - channels.resize(3); - channels[0].resize(n_rows*n_cols); - channels[1].resize(n_rows*n_cols); - channels[2].resize(n_rows*n_cols); - - images[i].resize(n_rows*n_cols*1); - for(int ch = 0; ch < 3; ++ch) - { - KL::UInt32 count = 0; - for(int r = 0; r < n_rows; ++r) - { - for(int c = 0; c < n_cols; ++c) - { - unsigned char temp = 0; - file.read((char*) &temp, sizeof(temp)); - channels[ch][count] = KL::Float64(temp); - //images[i][count] = KL::Float64(res); - count++; - //cerr << (int)temp << " " << res << endl; - } - } - } - - KL::UInt32 count = 0; - for(int r = 0; r < n_rows; ++r) - { - for(int c = 0; c < n_cols; ++c) - { - images[i][count] = (channels[0][count] + channels[1][count] + channels[2][count])/3.0; - //images[i][count] = ((int)images[i][count] / 255.0) * (scale_max - scale_min) + scale_min; - count ++; - } - } - } - } -} - - -FABRIC_EXT_EXPORT void MkCIFAR_parseBatch( - KL::MkCIFAR::INParam expr, - KL::String::INParam path, - KL::VariableArray >::IOParam images, - KL::VariableArray::IOParam labels) -{ - ReadBatch(string(path.data()), images, labels); -} - -/* -#define elif else if -using namespace cv; -using namespace std; - -Mat concatenateMat(vector &vec) { - - int height = vec[0].rows; - int width = vec[0].cols; - Mat res = Mat::zeros(height * width, vec.size(), CV_64FC1); - for(int i=0; i &vec) { - - int height = vec[0].rows; - int width = vec[0].cols; - Mat res = Mat::zeros(height * width * 3, vec.size(), CV_64FC1); - for(int i=0; i chs; - split(img, chs); - for(int j = 0; j < 3; j++) - { - Mat ptmat = chs[j].reshape(0, height * width); - Rect roi = cv::Rect(i, j * ptmat.rows, ptmat.cols, ptmat.rows); - Mat subView = res(roi); - ptmat.copyTo(subView); - } - } - divide(res, 255.0, res); - return res; -} - -void read_CIFAR10( - Mat &trainX, - Mat &testX, - Mat &trainY, - Mat &testY) -{ - string filename; - filename = "cifar-10-batches-bin/data_batch_1.bin"; - vector batch1; - Mat label1 = Mat::zeros(1, 10000, CV_64FC1); - read_batch(filename, batch1, label1); - - filename = "cifar-10-batches-bin/data_batch_2.bin"; - vector batch2; - Mat label2 = Mat::zeros(1, 10000, CV_64FC1); - read_batch(filename, batch2, label2); - - filename = "cifar-10-batches-bin/data_batch_3.bin"; - vector batch3; - Mat label3 = Mat::zeros(1, 10000, CV_64FC1); - read_batch(filename, batch3, label3); - - filename = "cifar-10-batches-bin/data_batch_4.bin"; - vector batch4; - Mat label4 = Mat::zeros(1, 10000, CV_64FC1); - read_batch(filename, batch4, label4); - - filename = "cifar-10-batches-bin/data_batch_5.bin"; - vector batch5; - Mat label5 = Mat::zeros(1, 10000, CV_64FC1); - read_batch(filename, batch5, label5); - - filename = "cifar-10-batches-bin/test_batch.bin"; - vector batcht; - Mat labelt = Mat::zeros(1, 10000, CV_64FC1); - read_batch(filename, batcht, labelt); - - Mat mt1 = concatenateMat(batch1); - Mat mt2 = concatenateMat(batch2); - Mat mt3 = concatenateMat(batch3); - Mat mt4 = concatenateMat(batch4); - Mat mt5 = concatenateMat(batch5); - Mat mtt = concatenateMat(batcht); - - Rect roi = cv::Rect(mt1.cols * 0, 0, mt1.cols, trainX.rows); - Mat subView = trainX(roi); - mt1.copyTo(subView); - roi = cv::Rect(label1.cols * 0, 0, label1.cols, 1); - subView = trainY(roi); - label1.copyTo(subView); - - roi = cv::Rect(mt1.cols * 1, 0, mt1.cols, trainX.rows); - subView = trainX(roi); - mt2.copyTo(subView); - roi = cv::Rect(label1.cols * 1, 0, label1.cols, 1); - subView = trainY(roi); - label2.copyTo(subView); - - roi = cv::Rect(mt1.cols * 2, 0, mt1.cols, trainX.rows); - subView = trainX(roi); - mt3.copyTo(subView); - roi = cv::Rect(label1.cols * 2, 0, label1.cols, 1); - subView = trainY(roi); - label3.copyTo(subView); - - roi = cv::Rect(mt1.cols * 3, 0, mt1.cols, trainX.rows); - subView = trainX(roi); - mt4.copyTo(subView); - roi = cv::Rect(label1.cols * 3, 0, label1.cols, 1); - subView = trainY(roi); - label4.copyTo(subView); - - roi = cv::Rect(mt1.cols * 4, 0, mt1.cols, trainX.rows); - subView = trainX(roi); - mt5.copyTo(subView); - roi = cv::Rect(label1.cols * 4, 0, label1.cols, 1); - subView = trainY(roi); - label5.copyTo(subView); - - mtt.copyTo(testX); - labelt.copyTo(testY); -} - -int main() { - Mat trainX, testX; - Mat trainY, testY; - trainX = Mat::zeros(1024, 50000, CV_64FC1); - testX = Mat::zeros(1024, 10000, CV_64FC1); - trainY = Mat::zeros(1, 50000, CV_64FC1); - testX = Mat::zeros(1, 10000, CV_64FC1); - - read_CIFAR10(trainX, testX, trainY, testY); - - return 0; -} -*/ \ No newline at end of file diff --git a/core/c++/cifar/MkCIFAR.fpm.json b/core/c++/cifar/MkCIFAR.fpm.json deleted file mode 100644 index 7dc37ce..0000000 --- a/core/c++/cifar/MkCIFAR.fpm.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "libs": "MkCIFAR", - "code": ["MkCIFAR.kl" ] -} diff --git a/core/c++/cifar/MkCIFAR.h b/core/c++/cifar/MkCIFAR.h deleted file mode 100644 index 6c5c165..0000000 --- a/core/c++/cifar/MkCIFAR.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_MkCIFAR__ -#define __KL2EDK_AUTOGEN_MkCIFAR__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'MkCIFAR.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" - -#include "MkCIFAR_functions.h" -#include "MkCIFAR_impl.h" -#include "MkCIFAR_type.h" -#include "global_functions.h" - -#endif // __KL2EDK_AUTOGEN_MkCIFAR__ diff --git a/core/c++/cifar/MkCIFAR.kl b/core/c++/cifar/MkCIFAR.kl deleted file mode 100644 index 18164ad..0000000 --- a/core/c++/cifar/MkCIFAR.kl +++ /dev/null @@ -1,20 +0,0 @@ -/**************************************************************************************************/ -/* */ -/* Informations : */ -/* This code is part of the project MLKL */ -/* */ -/* Contacts : */ -/* couet.julien@gmail.com */ -/* */ -/**************************************************************************************************/ - - -object MkCIFAR { - Data handle; -}; - -function MkCIFAR.parseBatch( - String path, - io Float64 images[][], - io UInt32 labels[]) -= "MkCIFAR_parseBatch"; diff --git a/core/c++/cifar/MkCIFAR_functions.h b/core/c++/cifar/MkCIFAR_functions.h deleted file mode 100644 index 88b4913..0000000 --- a/core/c++/cifar/MkCIFAR_functions.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_MkCIFAR_functions__ -#define __KL2EDK_AUTOGEN_MkCIFAR_functions__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'MkCIFAR_functions.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" -#include "MkCIFAR_type.h" - - -// Defined at MkCIFAR.kl:16:1 -FABRIC_EXT_EXPORT void MkCIFAR_parseBatch( - Fabric::EDK::KL::Traits< Fabric::EDK::KL::MkCIFAR >::INParam this_, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::String >::INParam path, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::Float64 > > >::IOParam images, - Fabric::EDK::KL::Traits< Fabric::EDK::KL::VariableArray< Fabric::EDK::KL::UInt32 > >::IOParam labels -); - -#endif // __KL2EDK_AUTOGEN_MkCIFAR_functions__ diff --git a/core/c++/cifar/MkCIFAR_impl.h b/core/c++/cifar/MkCIFAR_impl.h deleted file mode 100644 index 3d93138..0000000 --- a/core/c++/cifar/MkCIFAR_impl.h +++ /dev/null @@ -1,181 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_MkCIFAR_impl__ -#define __KL2EDK_AUTOGEN_MkCIFAR_impl__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'MkCIFAR_impl.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" - -#include "MkCIFAR.h" - -namespace Fabric { namespace EDK { namespace KL { - - struct MkCIFAR::Bits - { - ObjectCore __objectCore; - Data handle; - Object::Bits __interfaceObjectBits; - }; - - inline void MkCIFAR::ConstructEmpty( MkCIFAR *self ) - { - self->m_bits = 0; - } - - inline void MkCIFAR::ConstructCopy( MkCIFAR *self, MkCIFAR const *other ) - { - if ( (self->m_bits = other->m_bits) ) - AtomicUInt32Increment( &self->m_bits->__objectCore.refCount ); - } - - inline void MkCIFAR::AssignCopy( MkCIFAR *self, MkCIFAR const *other ) - { - if ( self->m_bits != other->m_bits ) - { - Destruct( self ); - ConstructCopy( self, other ); - } - } - - inline void MkCIFAR::Destruct( MkCIFAR *self ) - { - if ( self->m_bits - && AtomicUInt32DecrementAndGetValue( &self->m_bits->__objectCore.refCount ) == 0 ) - { - ObjectCore *objectCorePtr = &self->m_bits->__objectCore; - self->m_bits->__objectCore.lTableSwapPtrPtr->get()->lifecycleDestroy( - &objectCorePtr - ); - } - } - - inline MkCIFAR::MkCIFAR() - { - ConstructEmpty( this ); - } - - inline MkCIFAR MkCIFAR::Create() - { - static KL::SwapPtr const *sp = 0; - if ( !sp ) - { - sp = static_cast const *>( - s_callbacks.m_lookupGlobalSymbol( - "sp.function.kl.OO_MkCIFAR.createEmpty.io_AS0.OO_MkCIFAR", - 55 - ) - ); - if ( !sp ) - throwException( - "EDK internal error: failed to look up '%s'", - "sp.function.kl.OO_MkCIFAR.createEmpty.io_AS0.OO_MkCIFAR" - ); - } - void (*createFuncPtr)(void *) = ((void (*)(void *))sp->get()); - if ( !createFuncPtr ) - throwException( - "EDK internal error: target of '%s' is NULL", - "sp.function.kl.OO_MkCIFAR.createEmpty.io_AS0.OO_MkCIFAR" - ); - MkCIFAR result; - createFuncPtr( &result ); - return result; - } - - inline MkCIFAR::MkCIFAR( MkCIFAR const &that ) - { - ConstructCopy( this, &that ); - } - - inline MkCIFAR &MkCIFAR::operator =( MkCIFAR const &that ) - { - AssignCopy( this, &that ); - return *this; - } - - inline MkCIFAR::~MkCIFAR() - { - Destruct( this ); - } - - inline void MkCIFAR::appendDesc( String::IOParam string ) const - { - if ( m_bits ) - { - ObjectCore *objectCorePtr = &m_bits->__objectCore; - objectCorePtr->lTableSwapPtrPtr->get()->appendDesc( - &objectCorePtr, string - ); - } - else string.append( "null", 4 ); - } - inline uint32_t MkCIFAR::getRefCount() const - { - if ( m_bits ) - { - ObjectCore *objectCorePtr = &m_bits->__objectCore; - return objectCorePtr->refCount; - } - else return 0; - } - inline Type MkCIFAR::getType() const - { - if ( m_bits ) - { - return Type( m_bits->__objectCore.typeInfoSwapPtrPtr->get() ); - } - else return Type(); - } - - inline bool MkCIFAR::isValid() const - { - return !!m_bits; - } - - inline MkCIFAR::operator bool() const - { - return isValid(); - } - - inline bool MkCIFAR::operator !() const - { - return !isValid(); - } - - inline MkCIFAR::Bits *MkCIFAR::operator ->() - { - return static_cast( m_bits ); - } - - inline MkCIFAR::Bits const *MkCIFAR::operator ->() const - { - return static_cast( m_bits ); - } - - inline bool MkCIFAR::operator ==( INParam that ) - { - return m_bits == that.m_bits; - } - - inline bool MkCIFAR::operator !=( INParam that ) - { - return m_bits != that.m_bits; - } - - // Methods for 'Object' - -}}} - -#endif // __KL2EDK_AUTOGEN_MkCIFAR_impl__ diff --git a/core/c++/cifar/MkCIFAR_type.h b/core/c++/cifar/MkCIFAR_type.h deleted file mode 100644 index e306083..0000000 --- a/core/c++/cifar/MkCIFAR_type.h +++ /dev/null @@ -1,99 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_MkCIFAR_type__ -#define __KL2EDK_AUTOGEN_MkCIFAR_type__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'MkCIFAR_type.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" - -namespace Fabric { namespace EDK { namespace KL { - -// KL object 'MkCIFAR' -// Defined at MkCIFAR.kl:12:1 - -class MkCIFAR -{ -protected: - - struct Bits; - Bits *m_bits; - - friend struct Traits< MkCIFAR >; - - static void ConstructEmpty( MkCIFAR *self ); - - static void ConstructCopy( MkCIFAR *self, MkCIFAR const *other ); - - static void AssignCopy( MkCIFAR *self, MkCIFAR const *other ); - - static void Destruct( MkCIFAR *self ); - -public: - - typedef MkCIFAR &Result; - typedef MkCIFAR const &INParam; - typedef MkCIFAR &IOParam; - - MkCIFAR(); - - static MkCIFAR Create(); - - MkCIFAR( MkCIFAR const &that ); - - MkCIFAR &operator =( MkCIFAR const &that ); - - ~MkCIFAR(); - - void appendDesc( String::IOParam string ) const; - uint32_t getRefCount() const; - Type getType() const; - - bool isValid() const; - - operator bool() const; - - bool operator !() const; - - Bits *operator ->(); - - Bits const *operator ->() const; - - bool operator ==( INParam that ); - - bool operator !=( INParam that ); - - // Methods for 'Object' -}; - -inline void Traits::ConstructEmpty( MkCIFAR &val ) -{ - MkCIFAR::ConstructEmpty( &val ); -} -inline void Traits::ConstructCopy( MkCIFAR &lhs, MkCIFAR const &rhs ) -{ - MkCIFAR::ConstructCopy( &lhs, &rhs ); -} -inline void Traits::AssignCopy( MkCIFAR &lhs, MkCIFAR const &rhs ) -{ - MkCIFAR::AssignCopy( &lhs, &rhs ); -} -inline void Traits::Destruct( MkCIFAR &val ) -{ - MkCIFAR::Destruct( &val ); -} - -}}} - -#endif // __KL2EDK_AUTOGEN_MkCIFAR_type__ diff --git a/core/c++/cifar/SConstruct b/core/c++/cifar/SConstruct deleted file mode 100644 index 9ed07b8..0000000 --- a/core/c++/cifar/SConstruct +++ /dev/null @@ -1,38 +0,0 @@ -#################################################################################################### -# # -# Informations : # -# This code is part of the project MLKL # -# # -# Contacts : # -# couet.julien@gmail.com # -# # -#################################################################################################### - -import os, re, sys, subprocess -from sys import platform as _platform - -try: - fabricEDKPath = os.environ['FABRIC_DIR'] -except: - print "You must set FABRIC_DIR in your environment." - print "Refer to README.txt for more information." - sys.exit(1) -SConscript(os.path.join(fabricEDKPath, 'Samples', 'EDK', 'SConscript')) -Import('fabricBuildEnv') - -# Use of this flags to have access to C++11 -flags = { - 'CPPPATH': ['C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include'], - 'LIBPATH': [] -} -flags['CPPFLAGS'] = ['/O2'] - -fabricBuildEnv.MergeFlags(flags) -fabricBuildEnv.Extension( - 'MkCIFAR', - [ - 'MkCIFAR.cpp', 'MkCIFAR.kl' - ]) - - - \ No newline at end of file diff --git a/core/c++/cifar/aliases.h b/core/c++/cifar/aliases.h deleted file mode 100644 index ee3864c..0000000 --- a/core/c++/cifar/aliases.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_aliases__ -#define __KL2EDK_AUTOGEN_aliases__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'aliases.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -namespace Fabric { namespace EDK { namespace KL { - -}}} - -#endif // __KL2EDK_AUTOGEN_aliases__ diff --git a/core/c++/cifar/global.h b/core/c++/cifar/global.h deleted file mode 100644 index 3c4cb33..0000000 --- a/core/c++/cifar/global.h +++ /dev/null @@ -1,193 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_global__ -#define __KL2EDK_AUTOGEN_global__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'global.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -// Dependencies on other extensions -#define FABRIC_EDK_EXT_MkCIFAR_DEPENDENT_EXTS \ - { \ - { 0, 0, 0, 0, 0 } \ - } - -// forward declarations -namespace Fabric { namespace EDK { namespace KL { - class Object; - class MkCIFAR; -}}} - -#include "aliases.h" - -namespace Fabric { namespace EDK { namespace KL { - -// KL interface 'Object' -// Defined at (internal) - -class Object -{ -public: - - struct VTable - { - }; - - struct Bits - { - ObjectCore *objectCorePtr; - SwapPtr const *vTableSwapPtrPtr; - } *m_bits; - -protected: - - friend struct Traits< Object >; - - static void ConstructEmpty( Object *self ) - { - self->m_bits = 0; - } - - static void ConstructCopy( Object *self, Object const *other ) - { - if ( (self->m_bits = other->m_bits) ) - AtomicUInt32Increment( &self->m_bits->objectCorePtr->refCount ); - } - - static void AssignCopy( Object *self, Object const *other ) - { - if ( self->m_bits != other->m_bits ) - { - Destruct( self ); - ConstructCopy( self, other ); - } - } - - static void Destruct( Object *self ) - { - if ( self->m_bits - && AtomicUInt32DecrementAndGetValue( &self->m_bits->objectCorePtr->refCount ) == 0 ) - { - self->m_bits->objectCorePtr->lTableSwapPtrPtr->get()->lifecycleDestroy( - &self->m_bits->objectCorePtr - ); - } - } - -public: - - typedef Object &Result; - typedef Object const &INParam; - typedef Object &IOParam; - - Object() - { - ConstructEmpty( this ); - } - - Object( Object const &that ) - { - ConstructCopy( this, &that ); - } - - Object &operator =( Object const &that ) - { - AssignCopy( this, &that ); - return *this; - } - - ~Object() - { - Destruct( this ); - } - - void appendDesc( String::IOParam string ) const - { - if ( m_bits ) - m_bits->objectCorePtr->lTableSwapPtrPtr->get()->appendDesc( &m_bits->objectCorePtr, string ); - else string.append( "null", 4 ); - } - - bool isValid() const - { - return !!m_bits; - } - - operator bool() const - { - return isValid(); - } - - bool operator !() const - { - return !isValid(); - } - - bool operator ==( INParam that ) - { - return m_bits == that.m_bits; - } - - bool operator !=( INParam that ) - { - return m_bits != that.m_bits; - } - -}; - -template<> -struct Traits< Object > -{ - typedef Object &Result; - typedef Object const &INParam; - typedef Object &IOParam; - - static void ConstructEmpty( Object &val ); - static void ConstructCopy( Object &lhs, Object const &rhs ); - static void AssignCopy( Object &lhs, Object const &rhs ); - static void Destruct( Object &val ); -}; - -inline void Traits::ConstructEmpty( Object &val ) -{ - Object::ConstructEmpty( &val ); -} -inline void Traits::ConstructCopy( Object &lhs, Object const &rhs ) -{ - Object::ConstructCopy( &lhs, &rhs ); -} -inline void Traits::AssignCopy( Object &lhs, Object const &rhs ) -{ - Object::AssignCopy( &lhs, &rhs ); -} -inline void Traits::Destruct( Object &val ) -{ - Object::Destruct( &val ); -} - -template<> -struct Traits< MkCIFAR > -{ - typedef MkCIFAR &Result; - typedef MkCIFAR const &INParam; - typedef MkCIFAR &IOParam; - - static void ConstructEmpty( MkCIFAR &val ); - static void ConstructCopy( MkCIFAR &lhs, MkCIFAR const &rhs ); - static void AssignCopy( MkCIFAR &lhs, MkCIFAR const &rhs ); - static void Destruct( MkCIFAR &val ); -}; - -}}} - -#endif // __KL2EDK_AUTOGEN_global__ diff --git a/core/c++/cifar/global_functions.h b/core/c++/cifar/global_functions.h deleted file mode 100644 index 809656f..0000000 --- a/core/c++/cifar/global_functions.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef __KL2EDK_AUTOGEN_global_functions__ -#define __KL2EDK_AUTOGEN_global_functions__ - -#ifdef KL2EDK_INCLUDE_MESSAGES - #pragma message ( "Including 'global_functions.h'" ) -#endif - -//////////////////////////////////////////////////////////////// -// THIS FILE IS AUTOMATICALLY GENERATED -- DO NOT MODIFY!! -//////////////////////////////////////////////////////////////// -// Generated by kl2edk version 1.15.2 -//////////////////////////////////////////////////////////////// - -#include -#if FABRIC_EDK_VERSION_MAJ != 1 || FABRIC_EDK_VERSION_MIN != 15 -# error "This file needs to be rebuilt for the current EDK version!" -#endif - -#include "global.h" - - -#endif // __KL2EDK_AUTOGEN_global_functions__ diff --git a/core/cnn/MkCNNConfig.kl b/core/cnn/MkCNNConfig.kl new file mode 100644 index 0000000..6992594 --- /dev/null +++ b/core/cnn/MkCNNConfig.kl @@ -0,0 +1,668 @@ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// + +require Util, FileIO, MLKL; + + +/** + The AlembicArchiveReader is a wrapper for the AlembicIArchive. + It provides access to the higher level reader objects such as the AlembicXformReader. + \example + + require MLKL; + + operator entry() {} + + \endexample +*/ + +//************************************************************************************************// +// Activation functions // +// Struct to temporary hold the layers' parameters +struct MkCNNLayerParams { + String name; + Float32 epsW; + Float32 epsB; + Float32 momW; + Float32 momB; + Float32 wc; +}; + +function MkCNNLayerParams() { + this.name = ""; + this.epsW = 0.0; + this.epsB = 0.0; + this.momW = 0.0; + this.momB = 0.0; + this.wc = 0.0; +} + +/// Parse the layers configuration parameter relative to the neuron function +inline Boolean ParseNeuron(String line, io SInt32 neuron){ + if(line == "id") { + neuron = MK_NEURON_ID; + return true; + } + else if(line == "sig") { + neuron = MK_NEURON_SIG; + return true; + } + else if(line == "relu") { + neuron = MK_NEURON_RELU; + return true; + } + else if(line == "tanh") { + neuron = MK_NEURON_TANH; + return true; + } + else return false; +} + +/// Parse a string input +inline String ParseStr(String template, String line) { + return line.subString(line.find(template) + String(template).length(), line.length()); +} + +/// Parse a interger input +inline SInt32 ParseInt(String template, String line) { + String str = ParseStr(template, line); + return SInt32(str.toInteger()); +} + +/// Parse a scalar input +inline Float32 ParseScalar(String template, String line) { + String str = ParseStr(template, line); + return Float32(str.toScalar()); +} + +/// Check a interger input parameter +inline Boolean CheckInt(String name,, SInt32 val) { + if(val < 0) { + report("Error : parameters \"" + name + "\" is not set, add #" + name + "=val"); + return false; + } + return true; +} + +/// Check a scalar input parameter +inline Boolean CheckScalar(String name, Float32 val) { + if(val < 0) { + report("Error : parameters \"" + name + "\" is not set, add #" + name + "=val"); + return false; + } + return true; +} + +/// Check a path input parameter +inline Boolean CheckPath(String name, String path) { + if(!FilePath(path).exists() || (path.length == 0) ) { + report("Error : parameters \"" + name + "\" path " + path + " doesn't exist"); + return false; + } + return true; +} + +/// Parse the layers configuration and create them +inline Boolean ParseLayerPooling(String name, io TextReader reader, io MkCNNLayer layers[]) { + + SInt32 pool = -1; + SInt32 neuron = -1; + SInt32 filters = -1; + SInt32 in_channels = -1; + SInt32 pooling_size = -1; + Float32 init_w = -1.0; + Float32 init_b = -1.0; + + String line = reader.readLine(); + while(line.length == 0) line = reader.readLine(); + + while(line.length > 0) + { + if(line.find("pool=") > -1) { + String str = ParseStr("pool=", line); + pool = (str == "avg") ? 0 : (str == "max") ? 1 : -1; + } + else if(line.find("neuron=") > -1) ParseNeuron(ParseStr("neuron=", line), neuron); + else if(line.find("filters=") > -1) filters = ParseInt("filters=", line); + else if(line.find("inChannels=") > -1) in_channels = ParseInt("inChannels=", line); + else if(line.find("poolingSize=") > -1) pooling_size = ParseInt("poolingSize=", line); + else if(line.find("initW=") > -1) init_w = ParseScalar("initW=", line); + else if(line.find("initB=") > -1) init_b = ParseScalar("initB=", line); + line = reader.readLine(); + while(line[0] == "#") line = reader.readLine(); + } + + report("\n--- Pooling " + name + " ---"); + + if(!CheckInt("filters", filters)) return false; + else if(!CheckInt("pool", pool)) return false; + else if(!CheckInt("neuron", neuron)) return false; + else if(!CheckInt("inChannels", in_channels)) return false; + else if(!CheckInt("poolingSize", pooling_size)) return false; + else { + String pool_temp = (pool == 0) ? "avg" : "max"; + report("pool : " + pool_temp); + report("neuron : " + MkCNNNeuronModeAsStr(neuron)); + report("inSize : " + filters); + report("inChannels : " + in_channels); + report("poolingSize : " + pooling_size); + report("initW : " + init_w); + report("initB : " + init_b); + + if(pool == 0) + layers.push(MkCNNLayerAveragePooling( + name, neuron, filters, filters, + in_channels, pooling_size, init_w, init_b)); + else + layers.push(MkCNNLayerMaxPooling( + name, neuron, filters, filters, + in_channels, pooling_size, init_w, init_b)); + + return true; + } +} + +/// Parse the layers configuration and create them +inline Boolean ParseLayerConvolutional(String name, MkCNNLayerParams param, io TextReader reader, io MkCNNLayer layers[]) { + + SInt32 neuron = -1; + SInt32 filters = -1; + SInt32 in_channels = -1; + SInt32 filter_size = -1; + SInt32 out_channels = -1; + Float32 init_w = -1.0; + Float32 init_b = -1.0; + + String line = reader.readLine(); + while(line.length == 0) line = reader.readLine(); + + while(line.length > 0) + { + + if(line.find("neuron=") > -1) ParseNeuron(ParseStr("neuron=", line), neuron); + else if(line.find("filters=") > -1) filters = ParseInt("filters=", line); + else if(line.find("filterSize=") > -1) filter_size = ParseInt("filterSize=", line); + else if(line.find("inChannels=") > -1) in_channels = ParseInt("inChannels=", line); + else if(line.find("outChannels=") > -1) out_channels = ParseInt("outChannels=", line); + else if(line.find("initW=") > -1) init_w = ParseScalar("initW=", line); + else if(line.find("initB=") > -1) init_b = ParseScalar("initB=", line); + + line = reader.readLine(); + while(line[0] == "#") line = reader.readLine(); + } + + report("\n--- Convolutional " + name + " ---"); + + if(!CheckInt("filters", filters)) return false; + else if(!CheckInt("filterSize", filter_size)) return false; + else if(!CheckInt("neuron", neuron)) return false; + else if(!CheckInt("inChannels", in_channels)) return false; + else if(!CheckInt("outChannels", out_channels)) return false; + else { + report("neuron : " + ToConstLenght(MkCNNNeuronModeAsStr(neuron)) + "epsW : " + param.epsW); + report("filters : " + ToConstLenght(String(filters)) + "epsB : " + param.epsB); + report("filterSize : " + ToConstLenght(String(filter_size)) + "momW : " + param.momW); + report("inChannels : " + ToConstLenght(String(in_channels)) + "momB : " + param.momB); + report("outChannels : " + ToConstLenght(String(out_channels)) + "wc : " + param.wc); + report("initW : " + init_w); + report("initB : " + init_b); + + layers.push(MkCNNLayerConvolutional( + name, neuron, + filters, filters, filter_size, + in_channels, out_channels, init_w, init_b)); + + return true; + } +} + +/// Parse the layers configuration and create them +inline Boolean ParseLayerFully(String name, MkCNNLayerParams param, io TextReader reader, io MkCNNLayer layers[]) { + SInt32 neuron = -1; + SInt32 in_channels = -1; + SInt32 out_channels = -1; + Float32 init_w = -1.0; + Float32 init_b = -1.0;; + + String line = reader.readLine(); + while(line.length == 0) line = reader.readLine(); + + while(line.length > 0) + { + if(line.find("neuron=") > -1) ParseNeuron(ParseStr("neuron=", line), neuron); + else if(line.find("inChannels=") > -1) in_channels = ParseInt("inChannels=", line); + else if(line.find("outChannels=") > -1) out_channels = ParseInt("outChannels=", line); + else if(line.find("initW=") > -1) init_w = ParseScalar("initW=", line); + else if(line.find("initB=") > -1) init_b = ParseScalar("initB=", line); + line = reader.readLine(); + while(line[0] == "#") line = reader.readLine(); + } + + report("\n--- Fully-Connected " + name + " ---"); + + if(!CheckInt("neuron", neuron)) return false; + else if(!CheckInt("inChannels", in_channels)) return false; + else if(!CheckInt("outChannels", out_channels)) return false; + else { + report("neuron : " + ToConstLenght(MkCNNNeuronModeAsStr(neuron)) + "epsW : " + param.epsW); + report("inChannels : " + ToConstLenght(String(in_channels)) + "epsB : " + param.epsB); + report("outChannels : " + ToConstLenght(String(out_channels)) + "momW : " + param.momW); + report("initW : " + ToConstLenght(String(init_w)) + "momB : " + param.momB); + report("initB : " + ToConstLenght(String(init_b)) + "wc : " + param.wc); + + layers.push(MkCNNLayerFully(name, neuron, in_channels, out_channels, init_w, init_b)); + return true; + } +} + +/// Parse the layers configuration file. +/// Configure each layer and create them. +inline Boolean ParseLayersDefs(String path, MkCNNLayerParams params[], io MkCNNLayer layers[]) { + TextReader reader(); + + if(!reader.open(path)) + { + report("Error : Cannot open \"" + path + "\""); + return false; + } + + while(!reader.eof()) + { + String line = reader.readLine(); + + if(line.length > 0) + { + // Pass comment + while(line[0] == "#") line = reader.readLine(); + + // If we hit a new layer + if(line[0] == "[") + { + String name = line; + String layer_type = reader.readLine(); + + // If there is MkCNNLayerParams associated with this layer, get it back + MkCNNLayerParams param; + for(SInt32 p=0; p 0) + { + // Pass comment + while(line[0] == "#") line = reader.readLine(); + + // If we hit a new layer + if(line[0] == "[") + { + param.name = line; + line = reader.readLine(); + while(line.length > 0) + { + if(line.find("epsW=") > -1) param.epsW = ParseScalar("epsW=", line); + + if(line.find("epsB=") > -1) param.epsB = ParseScalar("epsB=", line); + + if(line.find("momW=") > -1) param.momW = ParseScalar("momW=", line); + + if(line.find("momB=") > -1) param.momB = ParseScalar("momB=", line); + + if(line.find("wc=") > -1) param.wc = ParseScalar("wc=", line); + + line = reader.readLine(); + } + + if(!CheckScalar("epsW", param.epsW)) return false; + + else if(!CheckScalar("epsB", param.epsB)) return false; + + else if(!CheckScalar("momW", param.momW)) return false; + + else if(!CheckScalar("momB", param.momB)) return false; + + else if(!CheckScalar("wc", param.wc)) return false; + + else + params.push(param); + } + } + } + + return reader.close(); +} +// Loss functions // +//************************************************************************************************// + + //*********************// + +//************************************************************************************************// +// Activation functions // +/// Class for network configuration +struct MkCNNConfig { + SInt32 worker; + SInt32 gpu; + SInt32 epoch; + SInt32 optimizer; + SInt32 batch_size; + SInt32 loss_function; + + String layers_defs_path; + String layers_params_path; + + String train_images_path; + String test_images_path; + String train_labels_path; + String test_labels_path; + + SInt32 save; + SInt32 auto_saving; + String main_save_dir_path; + String save_dir_path; + String save_file_name; + String load_file_path; +}; + +/// Default Constructor +function MkCNNConfig() { + this.worker = 1; + this.gpu = -1; + this.epoch = -1; + this.optimizer = -1; + this.batch_size = -1; + this.loss_function = -1; + this.save = -1; +} + +/// Parse the network configuration file +inline Boolean MkCNNConfig.parse!(String path) { + TextReader reader(); + + if(!reader.open(path)) + return false; + + while(!reader.eof()) + { + String line = reader.readLine(); + + if( (line.length() > 0) && line[0] != "#") + { + if(line.find("gpu=") > -1) this.gpu = ParseInt("gpu=", line); + else if(line.find("nbEpoch=") > -1) this.epoch = ParseInt("nbEpoch=", line); + + else if(line.find("batchSize=") > -1) this.batch_size = ParseInt("batch_size=", line); + + else if(line.find("optimizer=") > -1) this.optimizer = ParseInt("optimizer=", line); + + else if(line.find("lossFunction=") > -1) this.loss_function = ParseInt("lossFunction=", line); + + else if(line.find("layersDefsPath=") > -1) this.layers_defs_path = ParseStr("layersDefsPath=", line); + + else if(line.find("layersParamsPath=") > -1) this.layers_params_path = ParseStr("layersParamsPath=", line); + + else if(line.find("trainImagesPath=") > -1) this.train_images_path = ParseStr("trainImagesPath=", line); + + else if(line.find("testImagesPath=") > -1) this.test_images_path = ParseStr("testImagesPath=", line); + + else if(line.find("trainLabelsPath=") > -1) this.train_labels_path = ParseStr("trainLabelsPath=", line); + + else if(line.find("testLabelsPath=") > -1) this.test_labels_path = ParseStr("testLabelsPath=", line); + + else if(line.find("autoSaving=") > -1) this.auto_saving = ParseInt("autoSaving=", line); + + else if(line.find("save=") > -1) this.save = ParseInt("save=", line); + + else if(line.find("saveDirPath=") > -1) this.main_save_dir_path = ParseStr("saveDirPath=", line); + + else if(line.find("saveFileName=") > -1) this.save_file_name = ParseStr("saveFileName=", line); + + else if(line.find("loadFilePath=") > -1) this.load_file_path = ParseStr("loadFilePath=", line); + } + } + + if(!CheckInt("gpu", this.gpu)) return false; + + else if(!CheckInt("nbEpochs", this.epoch)) return false; + + else if(!CheckInt("batchSize", this.batch_size)) return false; + + else if(!CheckInt("optimizer", this.optimizer)) return false; + + else if(!CheckInt("lossFunction", this.loss_function)) return false; + + else if(!CheckPath("layersDefsPath", this.layers_defs_path)) return false; + + else if(!CheckPath("layersParamsPath", this.layers_params_path)) return false; + + else if(!CheckPath("trainImagesPath", this.train_images_path)) return false; + + else if(!CheckPath("testImagesPath", this.test_images_path)) return false; + + else if(!CheckInt("save", this.save)) return false; + + else if(!CheckPath("saveDirPath", this.main_save_dir_path)) return false; + + else { + report("1. Network parameters"); + report("tool* : " + "train"); + report("gpu : " + this.gpu); + report("nbEpochs : " + this.epoch); + report("batchSize : " + this.batch_size); + report("optimizer : " + MkCNNOptimizerModeAsStr(this.optimizer)); + report("lossFunction : " + MkCNNLossModeAsStr(this.loss_function)); + report("\n2. Layer definitions"); + report("layersDefs : " + this.layers_defs_path); + report("layerParams : " + this.layers_params_path); + report("\n3. Images and labels"); + report("trainImages : " + this.train_images_path); + report("testImages : " + this.test_images_path); + report("trainLabels* : " + this.train_labels_path); + report("testLabels* : " + this.test_labels_path); + report("\n4. Network saving and loading"); + report("save* : " + this.save); + report("autoSaving* : " + this.auto_saving); + report("saveDir : " + this.main_save_dir_path); + report("saveFileName* : " + this.save_file_name); + report("loadFilePath* : " + this.load_file_path); + } + + return reader.close(); +} + +/// To-Do +/// Check if the layers inputs and outputs sizes are correct +inline Boolean MkCNNConfig.check(io MkCNNLayer layers[]) { + return true; +} + +/// Create a directory to save the network +inline Boolean MkCNNConfig.initSaving!() { + + // Check if the main save folder exist + FileSystem file_system; + if(!file_system.exists(this.main_save_dir_path)) { + report("Error : directory \"saveDirPath\" doesn't exist"); + return false; + } + + // Get back the current time --> Name of the current folder saving + String date_time; + CurrentDateTime(date_time); + + // Create the current saving folder + FilePath file_path(this.main_save_dir_path); + file_path.append(FilePath(date_time)); + this.save_dir_path = file_path.string(); + file_system.createDirectory(file_path); + + return true; +} + +/// Configure the whole network form file, incuding the layers +function Boolean MkCNNConfig.config!(String path, io MkCNNLayer layers[]) { + + report("\n\n\n-------------------- Configuration --------------------"); + + report("\n------------ Network ------------\n"); + if(!this.parse(path)) + return false; + + report("\n\n------------ Layers ------------"); + MkCNNLayerParams layers_params[]; + if(!ParseLayersParams(this.layers_params_path, layers_params, layers)) return false; + if(!ParseLayersDefs(this.layers_defs_path, layers_params, layers)) return false; + if(!this.check(layers)) return false; + + if(this.save) return this.initSaving(); + return true; +} + +/// Save a layer +inline MkCNNConfig.saveLayer(io TextWriter writer, Ref layer) { + writer.writeLine("\nname=" + layer.name()); + writer.writeLine("mode="+ layer.modeAsStr()); + writer.writeLine("w="+ layer.weights()); + writer.writeLine("b="+ layer.bias()); +} + +/// Save the current network's layers +function Boolean MkCNNConfig.save!(MkCNNLayers layers) { + + // Check if the current saving file exists + // If not, create one by default + if(this.save_file_name.length() == 0) + this.save_file_name = "cnn_network.mlkl"; + + FilePath file_path(this.save_dir_path); + file_path.append(this.save_file_name); + + TextWriter writer(); + if(!writer.open(file_path.string())) { + report("Error : Cannot open \"" + file_path.string() + "\""); + return false; + } + + for(Index l=0; l layer) { + + String line = reader.readLine(); + + if(line.find("mode=") > -1) + line = reader.readLine(); + + if(line.find("w=") > -1) { + String data = ParseStr("w=", line); + data = data.subString(1, data.length()-2); + String datas[] = data.split(","); + + Float64 w[]; w.resize(datas.size()); + for(Index i=0; i -1 && scale_max > -1) + { + NormalizeImages(0.0, 1.0, data.train_images); + NormalizeImages(0.0, 1.0, data.test_images); + } + + return data; +} + +public Float64[][] LoadValidationCIFAR( + MkCNNConfig config, + Boolean rgb, + Float64 scale_min, + Float64 scale_max) +{ + Index validation_labels[]; + Float64 validation_images[][]; + MkCNNCpp cifar(); + cifar.readBatch_CIFAR(config.test_images_path, rgb, validation_images, validation_labels); + if(scale_min > -1 && scale_max > -1) + NormalizeImages(0.0, 1.0, validation_images); + return validation_images; +} + \ No newline at end of file diff --git a/core/cnn/MkCNNDropout.kl b/core/cnn/MkCNNDropout.kl new file mode 100644 index 0000000..de4706b --- /dev/null +++ b/core/cnn/MkCNNDropout.kl @@ -0,0 +1,174 @@ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// + +require MLKL; + + +/** + The AlembicArchiveReader is a wrapper for the AlembicIArchive. + It provides access to the higher level reader objects such as the AlembicXformReader. + \example + + require MLKL; + + operator entry() { + + } + + \endexample +*/ + +//************************************************************************************************// +// None filter // +/// Interface for filters +interface MkCNNFilter { + Index mode(); + mode!(Index mode); + Index context(); + context!(Index context); + Float64 dropoutRate(); + dropoutRate!(Float64 rate); + endBatch!(); + Float64[] filterForward!(Float64 outs[], Index index); + Float64[] filterBackward!(Float64 delta[], Index index); +}; + +/// Class for no-filter ~ base class +object MkCNNFilterNone : MkCNNFilter { + protected MkCNNDefs defs; + protected Index mode; + protected Index context; + protected Float64 dropout_rate; +}; + +/// Constructor +function MkCNNFilterNone(Index out_size) {} + +/// Return drop-out rate +public Float64 MkCNNFilterNone.dropoutRate() { + return this.dropout_rate; +} + +/// Set drop-out rate +public MkCNNFilterNone.dropoutRate!(Float64 rate) { + if(rate < 0.0 || rate >= 1.0) return; + this.dropout_rate = rate; +} + +/// Return the filter mode +public Index MkCNNFilterNone.mode() { + return this.mode; +} + +/// Return the filter mode +public MkCNNFilterNone.mode!(Index mode) { + this.mode = mode; +} + +/// Set the context +public Index MkCNNFilterNone.context() { + return this.context; +} + +/// Return the context +public MkCNNFilterNone.context!(Index context) { + this.context = context; +} + +/// \Internal +protected MkCNNFilterNone.shuffle!() {} + +public MkCNNFilterNone.endBatch!() {} + +/// Do nothing +public Float64[] MkCNNFilterNone.filterForward!(Float64 outs[], Index index) { + return outs; +} + +/// Do nothing +public Float64[] MkCNNFilterNone.filterBackward!(Float64 delta[], Index index) { + return delta; +} +// None filter // +//************************************************************************************************// + + //*********************// + +//************************************************************************************************// +// Drop-out filter // +const Index DROPOUT_CONTEXT_TRAIN_PHASE = 0; +const Index DROPOUT_CONTEXT_TEST_PHASE = 1; +const Index DROPOUT_MODE_PER_DATA = 0; +const Index DROPOUT_MODE_PER_BATCH = 1; + +/// Class for drop-out layer +object MkCNNDropout : MkCNNFilterNone { + private Index mask[]; + private Index out_size; + private Float64 masked_out[][]; + private Float64 masked_delta[][]; +}; + +/// Constructor +function MkCNNDropout(Index out_size) { + this.out_size = out_size; + this.context = DROPOUT_CONTEXT_TRAIN_PHASE; + this.mode = DROPOUT_MODE_PER_DATA; + this.dropout_rate = 0.5; + this.mask.resize(out_size); + + this.masked_out.resize(this.defs.taskSize()); + this.masked_delta.resize(this.defs.taskSize()); + for (Index i=0; i o, Index worker_size, Index batch_size) ; + updateWeights!(io Ref opti, Index worker_size, Index batch_size) ; divideHessian!(Index denominator); - Ref prev(); - prev!(Ref hs); - Ref next(); - next!(Ref hs); - Ref neuron(); - Float64[] fprop!(Float64 ins[], Index index); - Float64[] bprop!(Float64 current_delta[], Index index); - Float64[] bprop2nd!(Float64 current_delta2[]); + Ref prev(); + prev!(Ref hs); + Ref next(); + next!(Ref hs); + Ref neuron(); + Float64[] forward!(Float64 ins[], Index index); + Float64[] backward!(Float64 current_delta[], Index index); + Float64[] backward2!(Float64 current_delta2[]); }; /// Base class of all kind of NN layers -object MkCNNLayerBase : MkCNNLayerInterface { - protected MkCCNDefs defs; - protected String name; // Name of the layer - protected Index mode; // Layer's mode - protected Index in_size; // Layer input size - protected Index out_size; // Layer output size - protected Float64 init_w; // Initiliaze the weights with normal dist of std init_w - protected Float64 init_b; // Initiliaze the bias with normal dist of std init_b - protected Float64 w[]; // Weight vector - protected Float64 b[]; // Bias vector - protected Float64 dw[][]; // Difference of weight vector - protected Float64 db[][]; // Difference of bias vector - protected Float64 w_hessian[]; // Diagonal terms of the weight hessian matrix - protected Float64 b_hessian[]; // Diagonal terms of the bias hessian matrix - protected Float64 output[][]; // Last output of current layer, set by fprop - protected Float64 prev_delta[][]; // Last delta of previous layer, set by bprop - protected Float64 prev_delta2[]; // d^2E/da^2 - protected MkCNNNeuronInterface a; // Neuron function - protected Ref next; // Reference to the next layer, foward propagation - protected Ref prev; // Reference to the previous layer, backward propagation +object MkCNNLayerBase : MkCNNLayer { + protected MkCNNDefs defs; + protected String name; // Name of the layer + protected Index mode; // Layer's mode + protected Index in_size; // Layer input size + protected Index out_size; // Layer output size + protected Float64 init_w; // Initiliaze the weights with normal dist of std init_w + protected Float64 init_b; // Initiliaze the bias with normal dist of std init_b + protected Float64 w[]; // Weight vector + protected Float64 b[]; // Bias vector + protected Float64 dw[][]; // Difference of weight vector + protected Float64 db[][]; // Difference of bias vector + protected Float64 w_hessian[]; // Diagonal terms of the weight hessian matrix + protected Float64 b_hessian[]; // Diagonal terms of the bias hessian matrix + protected Float64 output[][]; // Last output of current layer, set by forward + protected Float64 prev_delta[][]; // Last delta of previous layer, set by backward + protected Float64 prev_delta2[]; // d^2E/da^2 + protected MkCNNNeuron neuron; // Neuron function + protected Ref next; // Reference to the next layer, foward propagation + protected Ref prev; // Reference to the previous layer, backward propagation }; /// Initilisation, called by the contructeurs @@ -139,19 +139,11 @@ protected MkCNNLayerBase.init!( this.init_w = init_w; this.init_b = init_b; - if(neuron == MK_NEURON_IDENTITY) - this.a = MkCNNNeuronIdentity(); - - else if(neuron == MK_NEURON_SIGMOID) - this.a = MkCNNNeuronSigmoid(); - - else if(neuron == MK_NEURON_RECTIFIEDLINEAR) - this.a = MkCNNNeuronRectifiedLinear(); - - else if(neuron == MK_NEURON_TANH) - this.a = MkCNNNeuronTanH(); - - else this.a = MkCNNNeuronIdentity(); + if(neuron == MK_NEURON_ID) this.neuron = MkCNNNeuronIdentity(); + else if(neuron == MK_NEURON_SIG) this.neuron = MkCNNNeuronSigmoid(); + else if(neuron == MK_NEURON_RELU) this.neuron = MkCNNNeuronRectifiedLinear(); + else if(neuron == MK_NEURON_TANH) this.neuron = MkCNNNeuronTanH(); + else this.neuron = MkCNNNeuronIdentity(); } /// Default constructor @@ -181,16 +173,17 @@ public MkCNNLayerBase( /// Display the class attributs public MkCNNLayerBase.display() { - report("\nMkCNNLayerBase Attributs"); - report("defs " + this.defs); - report("inSize " + this.in_size); - report("outSize " + this.out_size); - report("weightSize " + this.w.size()); - report("biasSize " + this.b.size()); - report("paramSize " + this.paramSize()); - report("fanInSize " + this.fanInSize()); - report("connectionSize " + this.connectionSize()); -} + report("\nLayer " + this.modeAsStr()); + report("name : " + this.name); + report("nuron : " + this.neuron.modeAsStr()); + report("inSize : " + this.in_size); + report("outSize : " + this.out_size); + report("weightSize : " + this.w.size()); + report("biasSize : " + this.b.size()); + report("paramSize : " + this.paramSize()); + report("fanInSize : " + this.fanInSize()); + report("connectionSize : " + this.connectionSize()); +} /// Return the layer name public String MkCNNLayerBase.name() { @@ -277,6 +270,16 @@ public MkCNNLayerBase.bias!(Float64 b[]) { this.b = b; } +/// Return the weights difference +public Float64[] MkCNNLayerBase.weightsDiff(Index index) { + return this.dw[index]; +} + +/// Return the bias difference +public Float64[] MkCNNLayerBase.biasDiff(Index index) { + return this.db[index]; +} + /// Return a the layer's output at a given worker_index protected Float64[] MkCNNLayerBase.output(Index index) { return this.output[index]; @@ -299,32 +302,32 @@ public Boolean MkCNNLayerBase.hasSameWeights(Ref other, Float64 } /// Return a pointer to the neuron function -public Ref MkCNNLayerBase.neuron() { - return this.a; +public Ref MkCNNLayerBase.neuron() { + return this.neuron; } /// Return a pointer to the next layer -public Ref MkCNNLayerBase.next() { +public Ref MkCNNLayerBase.next() { return this.next; } /// Return a pointer to the previous layer -public Ref MkCNNLayerBase.prev() { +public Ref MkCNNLayerBase.prev() { return this.prev; } /// Set the pointer to the next layer -private MkCNNLayerBase.next!(Ref layer) { +private MkCNNLayerBase.next!(Ref layer) { this.next = layer; } /// Set the pointer to the previous layer -private MkCNNLayerBase.prev!(Ref layer) { +private MkCNNLayerBase.prev!(Ref layer) { this.prev = layer; } /// Set the pointer to the next layer -public Boolean MkCNNLayerBase.connect!(io MkCNNLayerInterface tail) { +public Boolean MkCNNLayerBase.connect!(io MkCNNLayer tail) { if(this.outSize() != 0 && tail.inSize() != this.outSize()) { @@ -348,8 +351,10 @@ protected MkCNNLayerBase.clearDiff!(Index worker_size) { /// Weight layer initialisation, use uniform distribution by default public MkCNNLayerBase.initWeight!() { - Float64 weight_base = (this.init_w < 0) ? (0.5/sqrt(this.fanInSize())) : this.init_w; - Float64 bias_base = (this.init_b < 0) ? (0.5/sqrt(this.fanInSize())) : this.init_b; + // If the mean value for the weights (or bias) distribution is not set ( <=0 ) + // compte it by default (mainly for pooling layers) + Float64 weight_base = (this.init_w <= 0) ? (0.5/sqrt(this.fanInSize())) : this.init_w; + Float64 bias_base = (this.init_b <= 0) ? (0.5/sqrt(this.fanInSize())) : this.init_b; UniformRealDistribution(-weight_base, weight_base, this.w); UniformRealDistribution(-bias_base, bias_base, this.b); @@ -361,93 +366,77 @@ public MkCNNLayerBase.initWeight!() { /// Called after updating weight protected MkCNNLayerBase.postUpdate!() {} -/// When using several workers, merge the bias and weight differences +/// When using several workers (threads), merge the bias and weight differences protected MkCNNLayerBase.merge!(Index worker_size, Index batch_size) { - - //for (Index i=1; i o, - Index worker_size, - Index batch_size) -{ - if (this.w.size() == 0) - return; - +public MkCNNLayerBase.updateWeights!(io Ref opti, Index worker_size, Index batch_size) { + if (this.w.size() == 0) return; this.merge(worker_size, batch_size); - o.update(this.dw[0], this.w_hessian, this.w); - o.update(this.db[0], this.b_hessian, this.b); - + opti.update(this.uid(), this.dw[0], this.w_hessian, this.w); + opti.update(this.uid()*2, this.db[0], this.b_hessian, this.b); this.clearDiff(worker_size); this.postUpdate(); } /// Normalization of the hessian -public MkCNNLayerBase.divideHessian!(Index denominator) { - for(Index i=0; i>>( +operator MkCNNLayerMaxPoolingForward_task<<>>( Float64 ins[], Index out2in[][], io Index out2in_max[], io Float64 output[]) { - Index in_index[] = out2in[i]; Float64 max_value = -1.79769e+308; @@ -608,27 +596,27 @@ operator MkCNNLayerMaxPoolingFprop_task<<>>( } /// Forward propagation -public Float64[] MkCNNLayerMaxPooling.fprop!(Float64 ins[], Index index) { +public Float64[] MkCNNLayerMaxPooling.forward!(Float64 ins[], Index index) { Index out2in[][] = this.out2in; Index out2in_max[] = this.out2in_max; Float64 output[] = this.output[index]; - //report("MkCNNLayerMaxPooling.fprop 1"); + //report("MkCNNLayerMaxPooling.forward 1"); - MkCNNLayerMaxPoolingFprop_task<<>>( + MkCNNLayerMaxPoolingForward_task<<>>( ins, out2in, out2in_max, output); - //report("MkCNNLayerMaxPooling.fprop 2"); + //report("MkCNNLayerMaxPooling.forward 2"); - return (this.next()!= null) ? this.next().fprop(output, index) : output; + return (this.next()!= null) ? this.next().forward(output, index) : output; } /// Parallel task for Backward propagation -operator MkCNNLayerMaxPoolingBprop_task<<>>( - Ref prev_h, +operator MkCNNLayerMaxPoolingBackward_task<<>>( + Ref prev_h, Index out2in_max[], Index in2out[][], Float64 prev_out[], @@ -640,15 +628,15 @@ operator MkCNNLayerMaxPoolingBprop_task<<>>( } /// Backward propagetion -public Float64[] MkCNNLayerMaxPooling.bprop!(Float64 current_delta[], Index index) { +public Float64[] MkCNNLayerMaxPooling.backward!(Float64 current_delta[], Index index) { - Ref prev_h = this.prev().neuron(); + Ref prev_h = this.prev().neuron(); Float64 prev_output[] = this.prev().output(index); Float64 prev_delta[] = this.prev_delta[index]; Index in2out[][] = this.in2out; Index out2in_max[] = this.out2in_max; - MkCNNLayerMaxPoolingBprop_task<<>>( + MkCNNLayerMaxPoolingBackward_task<<>>( prev_h, out2in_max, in2out, @@ -656,12 +644,12 @@ public Float64[] MkCNNLayerMaxPooling.bprop!(Float64 current_delta[], Index inde current_delta, prev_delta); - return this.prev().bprop(this.prev_delta[index], index); + return this.prev().backward(this.prev_delta[index], index); } /// Parallel task for 2nd Backward propagation -operator MkCNNLayerMaxPoolingBprop2nd_task<<>>( - Ref prev_h, +operator MkCNNLayerMaxPoolingBackward2_task<<>>( + Ref prev_h, Index out2in_max[], Index in2out[][], Float64 prev_out[], @@ -673,15 +661,15 @@ operator MkCNNLayerMaxPoolingBprop2nd_task<<>>( } /// 2nd Backward propagetion -public Float64[] MkCNNLayerMaxPooling.bprop2nd!(Float64 current_delta2[]) { +public Float64[] MkCNNLayerMaxPooling.backward2!(Float64 current_delta2[]) { - Ref prev_h = this.prev().neuron(); + Ref prev_h = this.prev().neuron(); Float64 prev_output[] = this.prev().output(0); Float64 prev_delta2[] = this.prev_delta2; Index in2out[][] = this.in2out; Index out2in_max[] = this.out2in_max; - MkCNNLayerMaxPoolingBprop2nd_task<<>>( + MkCNNLayerMaxPoolingBackward2_task<<>>( prev_h, out2in_max, in2out, @@ -689,25 +677,25 @@ public Float64[] MkCNNLayerMaxPooling.bprop2nd!(Float64 current_delta2[]) { current_delta2, prev_delta2); - return this.prev().bprop2nd(this.prev_delta2); + return this.prev().backward2(this.prev_delta2); } -/* Max-pooling Layer */ -/**************************************************************************************************/ +// Max-pooling Layer // +//************************************************************************************************// - /***********************/ + //*********************// -/**************************************************************************************************/ -/* Layers (Stack of Layer) */ +//************************************************************************************************// +// Layers (Stack of Layer) // /// Class representong a stack of connected layers object MkCNNLayers { - private MkCNNLayerInterface layers[]; - private MkCNNLayerInterface first; // Is constrcted as an InputLayer + private MkCNNLayer layers[]; + private MkCNNLayer first; // Is constrcted as an InputLayer }; -private MkCNNLayers.construct!(io MkCNNLayers rhs) { +private MkCNNLayers.construct!(io MkCNNLayers layers) { this.add(this.first); - for (Index i = 1; i < rhs.layers.size(); i++) - this.add(rhs.layers[i]); + for (Index i = 1; i < layers.layers.size(); i++) + this.add(layers.layers[i]); } /// Constructor @@ -723,8 +711,8 @@ public MkCNNLayers.display() { } /// Add a new layer to the stack -public MkCNNLayers.add!(io MkCNNLayerInterface new_tail) { - if(this.tail() != null) +public MkCNNLayers.add!(io MkCNNLayer new_tail) { + if(this.tail()) this.tail().connect(new_tail); this.layers.push(new_tail); } @@ -740,19 +728,19 @@ public Index MkCNNLayers.size() { } /// Return the reference to the first layer -public Ref MkCNNLayers.head() { +public Ref MkCNNLayers.head() { if(this.empty) return null; else return this.layers[0]; } /// Return the reference to the last layer -public Ref MkCNNLayers.tail() { +public Ref MkCNNLayers.tail() { if(this.empty()) return null; else return this.layers[this.layers.size() - 1]; } /// Return the reference to the layer at index -public Ref MkCNNLayers.at(Index index) { +public Ref MkCNNLayers.at(Index index) { if(this.empty() || (index < 0) || (index >= this.layers.size()) ) return null; else return this.layers[index]; } @@ -764,13 +752,9 @@ public MkCNNLayers.initWeight!() { } /// Update the layers weights, after each batch iteration -public MkCNNLayers.updateWeights!( - io Ref o, - Index worker_size, - Index batch_size) -{ +public MkCNNLayers.updateWeights!(io Ref opti, Index worker_size, Index batch_size) { for(Index l=0; l>>( - Ref h, +operator MkCNNLayerFullyForward_task<<>>( + Ref neuron, Index in_size, Index out_size, Float64 w[], @@ -86,21 +90,21 @@ operator MkCNNLayerFullyFprop_task<<>>( for(Index c=0; c h = this.neuron(); + Ref neuron = this.neuron(); Index in_size = this.in_size; Index out_size = this.out_size; Float64 w[] = this.w; Float64 b[] = this.b; Float64 output[] = this.output[index]; - MkCNNLayerFullyFprop_task<<>>( - h, + MkCNNLayerFullyForward_task<<>>( + neuron, in_size, out_size, w, @@ -108,13 +112,13 @@ public Float64[] MkCNNLayerFully.fprop!(Float64 ins[], Index index) { ins, output); - Float64 outs[] = this.filter.filterFProp(this.output[index], index); - return (this.next() != null) ? this.next().fprop(outs, index) : outs; + Float64 outs[] = this.filter.filterForward(this.output[index], index); + return (this.next() != null) ? this.next().forward(outs, index) : outs; } /// Parallalized task for Backward propagation -operator MkCNNLayerFullyBprop_task_1<<>>( - Ref prev_h, +operator MkCNNLayerFullyBackward_task_1<<>>( + Ref prev_neuron, Index out_size, Float64 prev_out[], Float64 w[], @@ -124,11 +128,11 @@ operator MkCNNLayerFullyBprop_task_1<<>>( prev_delta[c] = 0.0; for(Index r=0; r>>( +operator MkCNNLayerFullyBackward_task_2<<>>( Index in_size, Index out_size, Float64 prev_out[], @@ -142,9 +146,9 @@ operator MkCNNLayerFullyBprop_task_2<<>>( } /// Backward propagation -public Float64[] MkCNNLayerFully.bprop!(Float64 current_delta[], Index index) { - - Ref prev_h = this.prev().neuron(); +public Float64[] MkCNNLayerFully.backward!(Float64 current_delta[], Index index) { + + Ref prev_neuron = this.prev().neuron(); Index in_size = this.in_size; Index out_size = this.out_size; Float64 w[] = this.w; @@ -153,28 +157,28 @@ public Float64[] MkCNNLayerFully.bprop!(Float64 current_delta[], Index index) { Float64 db[] = this.db[index]; Float64 dw[] = this.dw[index]; - MkCNNLayerFullyBprop_task_1<<>>( - prev_h, + MkCNNLayerFullyBackward_task_1<<>>( + prev_neuron, out_size, prev_output, w, - this.filter.filterBProp(current_delta, index), + this.filter.filterBackward(current_delta, index), prev_delta); - MkCNNLayerFullyBprop_task_2<<>>( + MkCNNLayerFullyBackward_task_2<<>>( in_size, out_size, prev_output, - this.filter.filterBProp(current_delta, index), + this.filter.filterBackward(current_delta, index), dw, db); - return this.prev().bprop(this.prev_delta[index], index); + return this.prev().backward(this.prev_delta[index], index); } /// Parallalized task for 2nd Backward propagation -operator MkCNNLayerFullyBprop2nd_task<<>>( - Ref prev_h, +operator MkCNNLayerFullyBackward2_task<<>>( + Ref prev_neuron, Index out_size, Float64 w[], Float64 prev_out[], @@ -188,24 +192,24 @@ operator MkCNNLayerFullyBprop2nd_task<<>>( prev_delta2[c] += current_delta2[r] * w[c*out_size + r] * w[c*out_size + r]; w_hessian[c*out_size + r] += current_delta2[r] * prev_out[c] * prev_out[c]; } - prev_delta2[c] *= prev_h.df(prev_out[c]) * prev_h.df(prev_out[c]); + prev_delta2[c] *= prev_neuron.df(prev_out[c]) * prev_neuron.df(prev_out[c]); } /// 2nd Backward propagation -public Float64[] MkCNNLayerFully.bprop2nd!(Float64 current_delta2[]) { +public Float64[] MkCNNLayerFully.backward2!(Float64 current_delta2[]) { for (Index r=0; r prev_h = this.prev().neuron(); + Ref prev_neuron = this.prev().neuron(); Index out_size = this.out_size; Float64 w[] = this.w; Float64 prev_output[] = this.prev().output(0); Float64 prev_delta2[] = this.prev_delta2; Float64 w_hessian[] = this.w_hessian; - MkCNNLayerFullyBprop2nd_task<<>>( - prev_h, + MkCNNLayerFullyBackward2_task<<>>( + prev_neuron, out_size, w, prev_output, @@ -213,45 +217,47 @@ public Float64[] MkCNNLayerFully.bprop2nd!(Float64 current_delta2[]) { w_hessian, prev_delta2); - return this.prev().bprop2nd(this.prev_delta2); + return this.prev().backward2(this.prev_delta2); } -/* Fully-connected Layer */ -/**************************************************************************************************/ +// Fully-connected Layer // +//************************************************************************************************// - /***********************/ + //*********************// -/**************************************************************************************************/ -/* Drop-out Layer */ +//************************************************************************************************// +// Drop-out Layer // /// Class for average-pooling layer object MkCNNLayerDropout : MkCNNLayerFully {}; /// Constructor -public MkCNNLayerDropout( +private MkCNNLayerDropout.init!( + String name, Index neuron, Index in_size, Index out_size, - Index weight_size, - Index bias_size) + Float64 init_w, + Float64 init_b) { - this.init("", neuron, in_size, out_size, weight_size, bias_size, -1.0, -1.0); + this.parent.init(name, neuron, in_size, out_size, init_w, init_b); this.mode = MK_LAYER_DROPOUT; this.filter = MkCNNDropout(); } +/// Constructor +public MkCNNLayerDropout(Index neuron, Index in_size, Index out_size) { + this.init("", neuron, in_size, out_size, -1.0, -1.0); +} + /// Constructor public MkCNNLayerDropout( String name, Index neuron, Index in_size, Index out_size, - Index weight_size, - Index bias_size, Float64 init_w, Float64 init_b) { - this.parent.init(name, neuron, in_size, out_size, weight_size, bias_size, init_w, init_b); - this.mode = MK_LAYER_DROPOUT; - this.filter = MkCNNDropout(); + this.init(name, neuron, in_size, out_size, init_w, init_b); } /// Set the dropout rate @@ -267,5 +273,5 @@ public MkCNNLayerDropout.context!(Index ctx) { private MkCNNLayerDropout.postUpdate!() { this.filter.endBatch(); } -/* Drop-out Layer */ -/**************************************************************************************************/ +// Drop-out Layer // +//************************************************************************************************// diff --git a/core/kl/cnn/MkCNNLayerPartial.kl b/core/cnn/MkCNNLayerPartial.kl similarity index 78% rename from core/kl/cnn/MkCNNLayerPartial.kl rename to core/cnn/MkCNNLayerPartial.kl index 57914b4..e859352 100644 --- a/core/kl/cnn/MkCNNLayerPartial.kl +++ b/core/cnn/MkCNNLayerPartial.kl @@ -1,12 +1,10 @@ -/**************************************************************************************************/ -/* */ -/* Informations : */ -/* This code is part of the project MLKL */ -/* */ -/* Contacts : */ -/* couet.julien@gmail.com */ -/* */ -/**************************************************************************************************/ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// require MLKL; @@ -25,8 +23,8 @@ require MLKL; */ -/**************************************************************************************************/ -/* Partial-connected Layer */ +//************************************************************************************************// +// Partial-connected Layer // /// Class for partially-connected layer object MkCNNLayerPartial : MkCNNLayerBase { protected MkCNNConnection weight2io[]; // weight_id . [(in_id, out_id)] @@ -95,13 +93,12 @@ public MkCNNLayerPartial( /// Display the class attributs public MkCNNLayerPartial.display() { this.parent.display(); - report("\nMkCNNLayerPartial Attributs"); - report("scaleFactor " + this.scale_factor); - report("weight2ioSize " + this.weight2io.size()); - report("out2wiSize " + this.out2wi.size()); - report("in2woSize " + this.in2wo.size()); - report("bias2outSize " + this.bias2out.size()); - report("out2biasSize " + this.out2bias.size()); + report("scaleFactor : " + this.scale_factor); + report("weight2ioSize : " + this.weight2io.size()); + report("out2wiSize : " + this.out2wi.size()); + report("in2woSize : " + this.in2wo.size()); + report("bias2outSize : " + this.bias2out.size()); + report("out2biasSize : " + this.out2bias.size()); } /// Return the total number of no-null connections @@ -136,11 +133,7 @@ public Index MkCNNLayerPartial.fanInSize() { } /// -protected MkCNNLayerPartial.connectWeight!( - Index input_index, - Index output_index, - Index weight_index) -{ +protected MkCNNLayerPartial.connectWeight!(Index input_index, Index output_index, Index weight_index) { this.weight2io[weight_index].push(MkCNNIndexPair(input_index, output_index)); this.out2wi[output_index].push(MkCNNIndexPair(weight_index, input_index)); this.in2wo[input_index].push(MkCNNIndexPair(weight_index, output_index)); @@ -177,9 +170,9 @@ protected MkCNNLayerPartial.remap!() { } /// Parallalized task for Forward propagation -operator MkCNNLayerPartialFprop_task<<>>( +operator MkCNNLayerPartialForward_task<<>>( Float64 scale_factor, - Ref h, + Ref h, MkCNNConnection out2wi[], Float64 w[], Float64 b[], @@ -196,9 +189,9 @@ operator MkCNNLayerPartialFprop_task<<>>( } /// Forward propagation -public Float64[] MkCNNLayerPartial.fprop!(Float64 ins[], Index index) { +public Float64[] MkCNNLayerPartial.forward!(Float64 ins[], Index index) { - Ref h = this.neuron(); + Ref h = this.neuron(); MkCNNConnection out2wi[] = this.out2wi; Float64 scale_factor = this.scale_factor; Index out2bias[] = this.out2bias; @@ -206,7 +199,7 @@ public Float64[] MkCNNLayerPartial.fprop!(Float64 ins[], Index index) { Float64 b[] = this.b; Float64 output[] = this.output[index]; - MkCNNLayerPartialFprop_task<<>>( + MkCNNLayerPartialForward_task<<>>( scale_factor, h, out2wi, @@ -216,13 +209,13 @@ public Float64[] MkCNNLayerPartial.fprop!(Float64 ins[], Index index) { ins, output); - return (this.next() != null) ? this.next().fprop(this.output[index], index) : this.output[index]; + return (this.next() != null) ? this.next().forward(this.output[index], index) : this.output[index]; } /// Parallalized task for Backward propagation -operator MkCNNLayerPartialBprop_task_1<<>>( +operator MkCNNLayerPartialBackward_task_1<<>>( Float64 scale_factor, - Ref prev_h, + Ref prev_h, MkCNNConnection in2wo[], Float64 prev_out[], Float64 w[], @@ -237,7 +230,7 @@ operator MkCNNLayerPartialBprop_task_1<<>>( } /// Parallalized task for Backward propagation -operator MkCNNLayerPartialBprop_task_2<<>>( +operator MkCNNLayerPartialBackward_task_2<<>>( Float64 scale_factor, MkCNNConnection weight2io[], Float64 prev_out[], @@ -252,9 +245,9 @@ operator MkCNNLayerPartialBprop_task_2<<>>( } /// Backward propagation -public Float64[] MkCNNLayerPartial.bprop!(Float64 current_delta[], Index index) { +public Float64[] MkCNNLayerPartial.backward!(Float64 current_delta[], Index index) { - Ref prev_h = this.prev().neuron(); + Ref prev_h = this.prev().neuron(); Float64 scale_factor = this.scale_factor; MkCNNConnection in2wo[] = this.in2wo; MkCNNConnection weight2io[] = this.weight2io; @@ -263,7 +256,7 @@ public Float64[] MkCNNLayerPartial.bprop!(Float64 current_delta[], Index index) Float64 prev_output[] = this.prev().output(index); Float64 prev_delta[] = this.prev_delta[index]; - MkCNNLayerPartialBprop_task_1<<>>( + MkCNNLayerPartialBackward_task_1<<>>( scale_factor, prev_h, in2wo, @@ -272,7 +265,7 @@ public Float64[] MkCNNLayerPartial.bprop!(Float64 current_delta[], Index index) current_delta, prev_delta); - MkCNNLayerPartialBprop_task_2<<>>( + MkCNNLayerPartialBackward_task_2<<>>( scale_factor, weight2io, prev_output, @@ -287,11 +280,11 @@ public Float64[] MkCNNLayerPartial.bprop!(Float64 current_delta[], Index index) this.db[index][i] += diff; } - return this.prev().bprop(this.prev_delta[index], index); + return this.prev().backward(this.prev_delta[index], index); } /// Parallalized task for 2nd Backward propagation -operator MkCNNLayerPartialBprop2nd_task_1<<>>( +operator MkCNNLayerPartialBackward2_task_1<<>>( Float64 scale_factor, MkCNNConnection weight2io[], Float64 prev_out[], @@ -307,9 +300,9 @@ operator MkCNNLayerPartialBprop2nd_task_1<<>>( } /// Parallalized task for 2nd Backward propagation -operator MkCNNLayerPartialBprop2nd_task_2<<>>( +operator MkCNNLayerPartialBackward2_task_2<<>>( Float64 scale_factor, - Ref prev_h, + Ref prev_h, Float64 w[], MkCNNConnection in2wo[], Float64 prev_out[], @@ -324,9 +317,9 @@ operator MkCNNLayerPartialBprop2nd_task_2<<>>( } /// 2nd Backward propagation -public Float64[] MkCNNLayerPartial.bprop2nd!(Float64 current_delta2[]) { +public Float64[] MkCNNLayerPartial.backward2!(Float64 current_delta2[]) { - Ref prev_h = this.prev().neuron(); + Ref prev_h = this.prev().neuron(); Float64 scale_factor = this.scale_factor; MkCNNConnection in2wo[] = this.in2wo; MkCNNConnection weight2io[] = this.weight2io; @@ -335,14 +328,14 @@ public Float64[] MkCNNLayerPartial.bprop2nd!(Float64 current_delta2[]) { Float64 prev_output[] = this.prev().output(0); Float64 prev_delta2[] = this.prev_delta2; - MkCNNLayerPartialBprop2nd_task_1<<>>( + MkCNNLayerPartialBackward2_task_1<<>>( scale_factor, weight2io, prev_output, current_delta2, w_hessian); - MkCNNLayerPartialBprop2nd_task_2<<>>( + MkCNNLayerPartialBackward2_task_2<<>>( scale_factor, prev_h, w, @@ -359,15 +352,15 @@ public Float64[] MkCNNLayerPartial.bprop2nd!(Float64 current_delta2[]) { this.b_hessian[i] += diff; } - return this.prev().bprop2nd(this.prev_delta2); + return this.prev().backward2(this.prev_delta2); } -/* Partial-connected Layer */ -/**************************************************************************************************/ +// Partial-connected Layer // +//************************************************************************************************// - /***********************/ + //*********************// -/**************************************************************************************************/ -/* Average-pooling Layer */ +//************************************************************************************************// +// Average-pooling Layer // /// Class for average-pooling layer object MkCNNLayerAveragePooling : MkCNNLayerPartial { private MkCNNIndex3D ins; @@ -464,17 +457,16 @@ public MkCNNLayerAveragePooling( /// Display the class attributs public MkCNNLayerAveragePooling.display() { this.parent.display(); - report("\nMkCNNLayerAveragePooling Attributs"); - report("ins " + this.ins); - report("outs " + this.outs); + report("ins : " + this.ins); + report("outs : " + this.outs); } -/* Average-pooling Layer */ -/**************************************************************************************************/ +// Average-pooling Layer // +//************************************************************************************************// - /***********************/ + //*********************// -/**************************************************************************************************/ -/* Convolutional Layer */ +//************************************************************************************************// +// Convolutional Layer // /// Class for convolutional layer object MkCNNLayerConvolutional : MkCNNLayerPartial { private MkCNNIndex3D ins; @@ -618,11 +610,10 @@ public MkCNNLayerConvolutional( /// Display the class attributs public MkCNNLayerConvolutional.display() { this.parent.display(); - report("\nMkCNNLayerConvolutional Attributs"); - report("ins " + this.ins); - report("outs " + this.outs); - report("weight " + this.weight); - report("window_size " + this.window_size); + report("ins : " + this.ins); + report("outs : " + this.outs); + report("weight : " + this.weight); + report("window_size : " + this.window_size); } /// TO-DO @@ -666,5 +657,5 @@ public MkCNNLayerConvolutional.weightToImage() { */ // return image; } -/* Convolutional Layer */ -/**************************************************************************************************/ +// Convolutional Layer // +//************************************************************************************************// diff --git a/core/cnn/MkCNNNetwork.kl b/core/cnn/MkCNNNetwork.kl new file mode 100644 index 0000000..32fcce3 --- /dev/null +++ b/core/cnn/MkCNNNetwork.kl @@ -0,0 +1,430 @@ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// + +require MLKL; + + +/** + The AlembicArchiveReader is a wrapper for the AlembicIArchive. + It provides access to the higher level reader objects such as the AlembicXformReader. + \example + + require MLKL; + + operator entry() { + + } + + \endexample +*/ + +const Index MK_GRAD_CHECK_ALL = 0; +const Index MK_GRAD_CHECK_FIRST = 1; +const Index MK_GRAD_CHECK_RANDOM = 2; + + +/// Class for Convolution Neural-Network +object MkCNNNetwork { + private String name; + private MkCNNDefs defs; + private MkCNNLayers layers; + private MkCNNLoss loss_function; + private MkCNNOptimizer optimizer; +}; + +/// Initilisation, called by the contructeurs and derived classes +private MkCNNNetwork.init!(String name, MkCNNConfig config) { + this.name = name; + this.layers = MkCNNLayers(); + + if(config.loss_function == MK_LOSS_MSE) this.loss_function = MkCNNLossMSE(); + else if(config.loss_function == MK_LOSS_CE) this.loss_function = MkCNNLossCE(); + else this.loss_function = MkCNNLossMSE(); + + if(config.optimizer == MK_OPTIMIZER_GD) this.optimizer = MkCNNOptimizerGD(); + else if(config.optimizer == MK_OPTIMIZER_GDLM) this.optimizer = MkCNNOptimizerGDLM(); + else if(config.optimizer == MK_OPTIMIZER_AGD) this.optimizer = MkCNNOptimizerAGD(); + else if(config.optimizer == MK_OPTIMIZER_RMSP) this.optimizer = MkCNNOptimizerRMSP(); + else if(config.optimizer == MK_OPTIMIZER_MGD) this.optimizer = MkCNNOptimizerMGD(); + else this.optimizer = MkCNNOptimizerGD(); +} + +/// Constructor +public MkCNNNetwork(String name, MkCNNConfig config) { + this.init(name, config); +} + +/// Constructor +public MkCNNNetwork(MkCNNConfig config) { + this.init("", config); +} + +/// Constructor +public MkCNNNetwork(String name, MkCNNConfig config, io MkCNNLayer layers[]) { + this.init(name, config); + this.add(layers); +} + +/// Constructor +public MkCNNNetwork(MkCNNConfig config, io MkCNNLayer layers[]) { + this.init("", config); + this.add(layers); +} + +/// Display the netwok attributs +public MkCNNNetwork.display() { + report("\n\n\n-------------------- Network --------------------"); + report("Attributs"); + report("name : " + this.name); + report("gpu : " + this.defs.gpu()); + report("taskSize : " + this.defs.taskSize()); + report("loss_function : " + this.loss_function.modeAsStr()); + this.optimizer().display(); + this.layers.display(); +} + +/// Return the input dimension of whole networks +public Index MkCNNNetwork.inDim() { + return this.layers.head().inSize(); +} + +/// Return the output dimension of whole networks +public Index MkCNNNetwork.outDim() { + return this.layers.tail().outSize(); +} + +/// Return the netork name +public String MkCNNNetwork.name() { + return this.name; +} + +/// Return a pointer to the netork optimizer +public Ref MkCNNNetwork.optimizer() { + return this.optimizer; +} + +/// Add a new layer to the network +public MkCNNNetwork.add!(io MkCNNLayer layer) { + this.layers.add(layer); +} + +/// Add a stack pf new layers to the network +public MkCNNNetwork.add!(io MkCNNLayer layers[]) { + for(Index i=0; i= out_dim) return; + vec[i] = temp.clone(); + vec[i][t[batch_index + i]] = this.targetValueMax(); + } +} + +/// Train one batch +private MkCNNNetwork.trainOnce!(Index batch_index, Float64 ins[][], Float64 t[][], Index size) { + + if(size == 1) { + this.backward(this.forward(ins[0], 0), t[0], 0); + this.layers.updateWeights(this.optimizer(), 1, 1); + } + + else { + Index num_tasks = size < this.defs.taskSize() ? 1 : this.defs.taskSize(); + Index data_per_thread = size / num_tasks; + Index remaining = size; + + for (Index i = 0; i < num_tasks; i++) + { + Index num = (i == (num_tasks - 1)) ? remaining : data_per_thread; + for (Index j = 0; j < num; j++) + this.backward(this.forward(ins[batch_index + i*num + j], i), t[i*num + j], i); + remaining -= num; + } + this.layers.updateWeights(this.optimizer(), num_tasks, size); + } +} + +/// Overload, Train one batch with 1D label array +private MkCNNNetwork.trainOnce!(Index batch_index, Float64 ins[][], Index t[], Index size) { + Float64 v[][]; + this.label2Vector(batch_index, size, t, v); + this.trainOnce(batch_index, ins, v, size); +} + +/// Check if a link is canonical +private Boolean MkCNNNetwork.isCanonicalLink(Ref h, Ref e) { + if (h.mode() == MK_NEURON_SIG && e.mode() == MK_LOSS_CE) return true; + if (h.mode() == MK_NEURON_TANH && e.mode() == MK_LOSS_CE) return true; + if (h.mode() == MK_NEURON_ID && e.mode() == MK_LOSS_MSE) return true; + return false; +} + +/// Computation of the hessian +private MkCNNNetwork.computeHessian!(Float64 ins[][], Index size_hessian) { + Index size = Math_min(ins.size(), size_hessian); + for(Index i=0; i h = this.layers.tail().neuron(); + if (this.isCanonicalLink(h, this.loss_function)) + { + for (Index i = 0; i < this.outDim(); i++) + delta[i] = outs[i] - t[i]; + } + else + { + for (Index i = 0; i < this.outDim(); i++) + delta[i] = this.loss_function.df(outs[i], t[i]) * h.df(outs[i]); + } + + this.layers.tail().backward(delta, idx); +} + +/// 2nd Backward propagation, use for Hessian computation +private MkCNNNetwork.backward2!(Float64 outs[]) { + Float64 delta[]; delta.resize(this.outDim()); + + Ref h = this.layers.tail().neuron(); + if (this.isCanonicalLink(h, this.loss_function)) + { + for (Index i = 0; i < this.outDim(); i++) + delta[i] = this.targetValueMax() * h.df(outs[i]); + } + else + { + for (Index i = 0; i < this.outDim(); i++) + delta[i] = this.targetValueMax() * h.df(outs[i]) * h.df(outs[i]); // FIXME + } + + this.layers.tail().backward2(delta); +} + +private Float64 MkCNNNetwork.computeDelta!( + Float64 ins[][], + Float64 v[][], + Index data_size, + io Float64 w[], + io Float64 dw[], + Index check_index) +{ + Float64 delta = 1e-10; + for(Index i=0; i current = this.layers.head(); + while((current = current.next()) != null) + { + // ignore first input layer + Float64 w[] = current.weights(); + Float64 b[] = current.bias(); + Float64 dw[] = current.weightsDiff(0); + Float64 db[] = current.biasDiff(0); + + if (w.size() == 0) continue; + + switch (mode) + { + case MK_GRAD_CHECK_ALL: + for (Index i = 0; i < w.size(); i++) + { + if (this.computeDelta(ins, v, data_size, w, dw, i) > eps) + return false; + } + for (Index i = 0; i < b.size(); i++) + { + if (this.computeDelta(ins, v, data_size, b, db, i) > eps) + return false; + } + break; + + case MK_GRAD_CHECK_FIRST: + if (this.computeDelta(ins, v, data_size, w, dw, 0) > eps) + return false; + if (this.computeDelta(ins, v, data_size, b, db, 0) > eps) + false; + break; + + case MK_GRAD_CHECK_RANDOM: + for (Index i = 0; i < 10; i++) + { + Index index; UniformRand(0, w.size() - 1, index); + if (this.computeDelta(ins, v, data_size, w, dw, index) > eps) + return false; + } + for (Index i = 0; i < 10; i++) + { + Index index; UniformRand(0, b.size() - 1, index); + if (this.computeDelta(ins, v, data_size, b, db, index) > eps) + return false; + } + break; + + default: + report("unknown grad-check type"); + //throw nn_error("unknown grad-check type"); + break; + } + } + + return true; +} diff --git a/core/cnn/MkCNNOptimizer.kl b/core/cnn/MkCNNOptimizer.kl new file mode 100644 index 0000000..e175bc6 --- /dev/null +++ b/core/cnn/MkCNNOptimizer.kl @@ -0,0 +1,353 @@ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// + +require MLKL; + + +//************************************************************************************************// +// Optimizer // +const Index MK_OPTIMIZER_GD = 0; +const Index MK_OPTIMIZER_GDLM = 1; +const Index MK_OPTIMIZER_AGD = 2; +const Index MK_OPTIMIZER_RMSP = 3; +const Index MK_OPTIMIZER_MGD = 4; + +function String MkCNNOptimizerModeAsStr(Index mode) { + if(mode == MK_OPTIMIZER_GD) return "MK_OPTIMIZER_GD"; + else if(mode == MK_OPTIMIZER_GDLM) return "MK_OPTIMIZER_GDLM"; + else if(mode == MK_OPTIMIZER_AGD) return "MK_OPTIMIZER_AGD"; + else if(mode == MK_OPTIMIZER_RMSP) return "MK_OPTIMIZER_RMSP"; + else if(mode == MK_OPTIMIZER_MGD) return "MK_OPTIMIZER_MGD"; + else return "MK_OPTIMIZER_UNKNOWN"; +} + +/// Interface for optimizer +interface MkCNNOptimizer { + Index mode(); + String modeAsStr(); + update!(UInt64 layer_uid, Float64 dw[], Float64 h[], io Float64 w[]); + Boolean requiresHessian(); + learningRate!(Float64 learning_rate); + weigthDecay!(Float64 weigth_decay); + Float64 learningRate(); + Float64 weigthDecay(); + display(); + reset!(); +}; + +/// Base class for optimizer +object MkCNNOptimizerBase : MkCNNOptimizer { + protected Index mode; + Float64 learning_rate; // Learning rate + Float64 weigth_decay; // Weight decay +}; + +/// Init the optimizer value +protected MkCNNOptimizerBase.init!(Float64 learning_rate, Float64 weigth_decay) { + this.learning_rate = learning_rate; + this.weigth_decay = weigth_decay; +} + +/// Return the current mode, either MK_LOSS_CE or MK_LOSS_MSE +public Index MkCNNOptimizerBase.mode() { + return this.mode; +} + +/// Return the current mode as string +public String MkCNNOptimizerBase.modeAsStr() { + return MkCNNOptimizerModeAsStr(this.mode); +} + +/// Display the optimizer attributs +public MkCNNOptimizerBase.display() { + report("optimizer : " + this.modeAsStr()); + report("learning_rate : " + this.learning_rate); + report("weigthDecay : " + this.weigth_decay); +} + +/// Set the learning rate +public MkCNNOptimizerBase.learningRate!(Float64 learning_rate) { + this.learning_rate = learning_rate; +} + +/// Get the learning rate +public Float64 MkCNNOptimizerBase.learningRate() { + return this.learning_rate; +} + +/// Set the weigth decay +public MkCNNOptimizerBase.weigthDecay!(Float64 weigth_decay) { + this.weigth_decay = weigth_decay; +} + +/// Get the weigth decay +public Float64 MkCNNOptimizerBase.weigthDecay() { + return this.weigth_decay; +} + +/// Reset +public MkCNNOptimizerBase.reset!() {} + +/// Return TRUE if the optimizer needs an Hessian matrix +public Boolean MkCNNOptimizerBase.requiresHessian() { + return false; +} + +/// Update the optimizer +/// To-be overload +public MkCNNOptimizerBase.update!( + UInt64 layer_uid, + Float64 dw[], + Float64 h[], + io Float64 w[]) +{ +} +// Optimizer // +//************************************************************************************************// + + //*********************// + +//************************************************************************************************// +// Gradient Descente // +/// Class for gradient descente optimizer +object MkCNNOptimizerGD : MkCNNOptimizerBase {}; + +/// Constructor +public MkCNNOptimizerGD() { + this.init(0.01, 0.1); + this.mode = MK_OPTIMIZER_GD; +} + +public MkCNNOptimizerGD(Float64 learning_rate, Float64 weigth_decay) { + this.parent.init(learning_rate, weigth_decay); + this.mode = MK_OPTIMIZER_GD; +} + +operator MkCNNOptimizerGDUpdate_task<<>>(Ref opti, Float64 dw[], io Float64 w[]) { + w[i] = w[i] - opti.learningRate() * (dw[i] + opti.weigthDecay() * w[i]); +} + +public MkCNNOptimizerGD.update!( + UInt64 layer_uid, + Float64 dw[], + Float64 h[], + io Float64 w[]) +{ + MkCNNOptimizerGDUpdate_task<<>>(this, dw, w); +} +// Gradient Descente // +//************************************************************************************************// + + //*********************// + +//************************************************************************************************// +// Gradient Descente Levenbarq-Marquant // +/// Class for gradient descente Levenbarq-Marquant optimizer +object MkCNNOptimizerGDLM : MkCNNOptimizerBase {}; + +/// Constructor +public MkCNNOptimizerGDLM() { + // constant to prevent step size from becoming too large when H is small + this.init(0.00085, 0.02); + this.mode = MK_OPTIMIZER_GDLM; +} + +public MkCNNOptimizerGDLM(Float64 learning_rate, Float64 weigth_decay) { + this.init(learning_rate, weigth_decay); + this.mode = MK_OPTIMIZER_GDLM; +} + +operator MkCNNOptimizerGDLMUpdate_task<<>>( + Ref opti, + Float64 dw[], + Float64 h[], + io Float64 w[]) +{ + w[i] = w[i] - (opti.learningRate() / (h[i] + opti.weigthDecay()) ) * dw[i]; +} + +public MkCNNOptimizerGDLM.update!( + UInt64 layer_uid, + Float64 dw[], + Float64 h[], + io Float64 w[]) +{ + MkCNNOptimizerGDLMUpdate_task<<>>(this, dw, h, w); +} + +public Boolean MkCNNOptimizerGDLM.requiresHessian() { + return true; +} +// Gradient Descente Levenbarq-Marquant // +//************************************************************************************************// + + //*********************// + +//************************************************************************************************// +// Stateful optimizers // +/// Base class for optimizer +object MkCNNOptimizerStatefulBase : MkCNNOptimizerBase { + protected Float64 E[UInt64][]; + Float64 eps; +}; + +/// Init the optimizer value +protected MkCNNOptimizerStatefulBase.init!(Float64 learning_rate, Float64 weigth_decay) { + this.parent.init(learning_rate, weigth_decay); + this.eps = 1.e-8; +} + +public Float64[] MkCNNOptimizerStatefulBase.get!(UInt64 layer_uid, Float64 w[]) { + if(!this.E.has(layer_uid)) + { + Float64 temp[]; temp.resize(w.size()); + this.E.set(layer_uid, temp); + } + return this.E.get(layer_uid); +} + +public MkCNNOptimizerStatefulBase.set!(UInt64 layer_uid, Float64 g[]) { + if(this.E.has(layer_uid)) + this.E.set(layer_uid, g); +} + +public MkCNNOptimizerStatefulBase.reset!() { + this.E.clear(); +} +// Stateful optimizers // +//************************************************************************************************// + + //*********************// + +//************************************************************************************************// +// Adaptive Gradient Descente // +// J. Duchi, E. Hazan and Y. Singer, +// Adaptive subgradient methods for online learning and stochastic optimization +// The Journal of Machine Learning Research, pages 2121-2159, 2011. +// http://xcorr.net/2014/01/23/adagrad-eliminating-learning-rates-in-stochastic-gradient-descent/ +object MkCNNOptimizerAGD : MkCNNOptimizerStatefulBase {}; + +public MkCNNOptimizerAGD() { + this.parent.init(0.01, 0.0); + this.mode = MK_OPTIMIZER_AGD; +} + +public MkCNNOptimizerAGD(Float64 learning_rate) { + this.parent.init(learning_rate, 0.0); + this.mode = MK_OPTIMIZER_AGD; +} + +operator MkCNNOptimizerAGDUpdate_task<<>>( + Ref opti, + Float64 dw[], + io Float64 w[], + io Float64 g[]) +{ + g[i] += dw[i] * dw[i]; + w[i] = w[i] - opti.learning_rate * dw[i] / (sqrt(g[i]) + opti.eps); +} + +public MkCNNOptimizerAGD.update!( + UInt64 layer_uid, + Float64 dw[], + Float64 h[], + io Float64 w[]) +{ + Float64 g[] = this.get(layer_uid, w); + MkCNNOptimizerAGDUpdate_task<<>>(this, dw, w, g); + this.set(layer_uid, g); +} +// Adaptive Gradient Descente // +//************************************************************************************************// + + //*********************// + +//************************************************************************************************// +// RMS Prop // +// T. Tielemanand G.E. Hinton, +// Lecture 6.5 - rmsprop, COURSERA: Neural Networks for Machine Learning (2012) +object MkCNNOptimizerRMSP : MkCNNOptimizerStatefulBase {}; + +public MkCNNOptimizerRMSP() { + this.parent.init(0.0001, 0.99); + this.mode = MK_OPTIMIZER_RMSP; +} + +public MkCNNOptimizerRMSP(Float64 learning_rate, Float64 weigth_decay) { + this.parent.init(learning_rate, weigth_decay); + this.mode = MK_OPTIMIZER_RMSP; +} + +operator MkCNNOptimizerRMSPUpdate_task<<>>( + Ref opti, + Float64 dw[], + io Float64 w[], + io Float64 g[]) +{ + g[i] = opti.weigth_decay * g[i] + (1.0 - opti.weigth_decay) * dw[i] * dw[i]; + w[i] = w[i] - opti.learning_rate * dw[i] / (sqrt(g[i]) + opti.eps); +} + +public MkCNNOptimizerRMSP.update!( + UInt64 layer_uid, + Float64 dw[], + Float64 h[], + io Float64 w[]) +{ + Float64 g[] = this.get(layer_uid, w); + MkCNNOptimizerRMSPUpdate_task<<>>(this, dw, w, g); + this.set(layer_uid, g); +} +// RMS Prop // +//************************************************************************************************// + + //*********************// + +//************************************************************************************************// +// Adaptive Gradient Descente // +// SGD with Momentum, B T Polyak, +// Some methods of speeding up the convergence of iteration methods +// USSR Computational Mathematics and Mathematical Physics, 4(5):1-17, 1964. +object MkCNNOptimizerMGD : MkCNNOptimizerStatefulBase { + Float64 mu; // Momentum +}; + +public MkCNNOptimizerMGD() { + this.parent.init(0.01, 0.0); + this.mu = 0.9; + this.mode = MK_OPTIMIZER_MGD; +} + +public MkCNNOptimizerMGD(Float64 learning_rate, Float64 weigth_decay) { + this.parent.init(learning_rate, weigth_decay); + this.mu = 0.9; + this.mode = MK_OPTIMIZER_MGD; +} + +operator MkCNNOptimizerMGDUpdate_task<<>>( + Ref opti, + Float64 dw[], + io Float64 w[], + io Float64 g[]) +{ + g[i] = opti.mu * g[i] - opti.learning_rate * (dw[i] + w[i] * opti.weigth_decay); + w[i] += g[i]; +} + +public MkCNNOptimizerMGD.update!( + UInt64 layer_uid, + Float64 dw[], + Float64 h[], + io Float64 w[]) +{ + Float64 g[] = this.get(layer_uid, w); + MkCNNOptimizerMGDUpdate_task<<>>(this, dw, w, g); + this.set(layer_uid, g); +} +// Adaptive Gradient Descente // +//************************************************************************************************// \ No newline at end of file diff --git a/core/cnn/MkCNNUtils.kl b/core/cnn/MkCNNUtils.kl new file mode 100644 index 0000000..3a21175 --- /dev/null +++ b/core/cnn/MkCNNUtils.kl @@ -0,0 +1,313 @@ +//************************************************************************************************// +// // +// Code part of the project MLKL // +// // +// couet.julien@gmail.com // +// // +//************************************************************************************************// + +require MLKL; + + +//************************************************************************************************// +// Config // +struct MkCNNDefs { + Index task_size; + Boolean gpu; +}; + +function MkCNNDefs() { + this.task_size = 1; + this.gpu = false; +} + +function Index MkCNNDefs.taskSize() { + return this.task_size; +} + +function Boolean MkCNNDefs.gpu() { + return this.gpu; +} + +function Index MaxIndex(Float64 array[]) { + Float64 max_val = 0; + Index max_index = 0; + for(Index i=0; i > confusion_matrix; +}; + +function MkCNNNetworkResult(Index num_success, Index num_total) { + this.num_success = num_success; + this.num_total = num_total; +} + +function MkCNNNetworkResult() { + this.num_success = 0; + this.num_total = 0; +} + +function Float64 MkCNNNetworkResult.accuracy() { + return this.num_success * 100.0 / this.num_total; +} + +function MkCNNNetworkResult.printSummary() { + report("accuracy : " + this.accuracy() + "% (" + this.num_success + "/" + this.num_total); +} + +function MkCNNNetworkResult.printDetail() { + this.printSummary(); + /* + auto all_labels = labels(); + + os << std::setw(5) << "*" << " "; + for (auto c : all_labels) + os << std::setw(5) << c << " "; + os << std::endl; + + for (auto r : all_labels) { + os << std::setw(5) << r << " "; + for (auto c : all_labels) + os << std::setw(5) << confusion_matrix[r][c] << " "; + os << std::endl; + } + } + + std::set labels() const { + std::set all_labels; + for (auto r : confusion_matrix) { + all_labels.insert(r.first); + for (auto c : r.second) + all_labels.insert(c.first); + } + return all_labels; + } + */ +} + + +/// Display iteration info +struct MkCNNEnumBatch { + Index counter; // To update the display + Index training_size; // Number of training images + Index current_size; // Current image + Index batch_size; // Batch size +}; + +/// Constructor +function MkCNNEnumBatch(Index training_size, Index batch_size) { + report("\n\n\n-------------------- Training --------------------"); + this.training_size = training_size; + this.batch_size = batch_size; + this.current_size = 0; +} + +/// Reset the info +function MkCNNEnumBatch.reset!(Index ite, Index nb_epoch) { + report("Epoch " + Index(ite+1) + "/" + nb_epoch); + this.counter = 0; + this.current_size = 0; + this.display(); +} + +/// Display the current percentage of computation +function MkCNNEnumBatch.display() { + Float32 percent = 100.0 * this.batch_size * Float32(this.current_size) / Float32(this.training_size); + ReportR("Train : " + Index(percent) + "%"); +} + +/// Update the iteration info and display it +function MkCNNEnumBatch.update!() { + this.current_size++; + this.counter ++; + + // We don't want to write too much in the console. + // Depending on the batch size, this function can be called really often + if(this.counter == 50) { + this.counter = 0; + this.display(); + } +} + + +/// Structure to display epoch info +struct MkCNNEnumEpoch { + UInt64 start; // To compute the procesing time + Float64 learning_rate_decay; // Update the optimizer learning rate +}; + +/// Desfault constructor, set the learning rate to 0.85 +function MkCNNEnumEpoch() { + this.start = getCurrentTicks(); + this.learning_rate_decay = 0.85; +} + +/// Constructor +function MkCNNEnumEpoch(Float64 learning_rate_decay) { + this.start = getCurrentTicks(); + this.learning_rate_decay = learning_rate_decay; +} + +/// Display the epoch info and perfor a test +function MkCNNEnumEpoch.display(io Ref nn, Float64 test_images[][], Index test_labels[]) { + report("Train : 100%"); + Float32 tim = Float32(getSecondsBetweenTicks(this.start, getCurrentTicks())/60.0); + report("Time : " + tim + " mins"); + + MkCNNNetworkResult res = nn.test(test_images, test_labels); + Ref opti = nn.optimizer(); + report("Learning rate : " + Float32(opti.learningRate())); + Float32 succes = 100.0*Float32(res.num_success)/Float32(res.num_total); + report("Test : " + succes + "% succes\n"); +} + +/// Update the epoch info and the optimizer +function MkCNNEnumEpoch.update!(io Ref nn, Float64 test_images[][], Index test_labels[]) { + // Display the info + this.display(nn, test_images, test_labels); + + // Update the optimization learning rate + Ref opti = nn.optimizer(); + opti.learningRate(opti.learningRate()*this.learning_rate_decay); + opti.learningRate(Math_max(0.00001, opti.learningRate())); + this.start = getCurrentTicks(); +} +// Outputs // +//************************************************************************************************// \ No newline at end of file diff --git a/core/cnn/c++/.sconsign.dblite b/core/cnn/c++/.sconsign.dblite new file mode 100644 index 0000000..4109ef6 Binary files /dev/null and b/core/cnn/c++/.sconsign.dblite differ diff --git a/core/cnn/c++/MkCNNCpp-Windows-x86_64.exp b/core/cnn/c++/MkCNNCpp-Windows-x86_64.exp new file mode 100644 index 0000000..211ef31 Binary files /dev/null and b/core/cnn/c++/MkCNNCpp-Windows-x86_64.exp differ diff --git a/core/c++/MNIST/MkMNIST.cpp b/core/cnn/c++/MkCNNCpp.cpp similarity index 75% rename from core/c++/MNIST/MkMNIST.cpp rename to core/cnn/c++/MkCNNCpp.cpp index 2d4ddbe..5299e05 100644 --- a/core/c++/MNIST/MkMNIST.cpp +++ b/core/cnn/c++/MkCNNCpp.cpp @@ -21,13 +21,15 @@ #include using namespace std; -#include +#include #include using namespace Fabric::EDK; -IMPLEMENT_FABRIC_EDK_ENTRIES( MkMNIST ) +IMPLEMENT_FABRIC_EDK_ENTRIES( MkCNNCpp ) +/********/ + #define ReverseEndian(x) ReverseEndianFunc((unsigned char *) &x, sizeof(x)) void ReverseEndianFunc(unsigned char * b, int n) { int i = 0; @@ -39,48 +41,36 @@ void ReverseEndianFunc(unsigned char * b, int n) { } } -FABRIC_EXT_EXPORT void MkMNIST_parseLabels( - KL::VariableArray::IOParam labels, - KL::MkMNIST::INParam expr, - KL::String::INParam path) -{ - - ifstream ifs(path.data(), ios::in | ios::binary); - if (ifs.bad() || ifs.fail()) - { - cerr << "Error MkMNIST_parseLabels : image-file error" << endl; - return; - } - +struct mnist_header { uint32_t magic_number; uint32_t num_items; - ifs.read((char*) &magic_number, 4); - ifs.read((char*) &num_items, 4); - ReverseEndian(magic_number); - ReverseEndian(num_items); - if (magic_number != 0x00000801 || num_items <= 0) + uint32_t num_rows; + uint32_t num_cols; +}; + +inline void ReadImageHeader_MNIST(ifstream& ifs, mnist_header& header) { + + ifs.read((char*) &header.magic_number, 4); + ifs.read((char*) &header.num_items, 4); + ifs.read((char*) &header.num_rows, 4); + ifs.read((char*) &header.num_cols, 4); + ReverseEndian( header.magic_number); + ReverseEndian( header.num_items); + ReverseEndian( header.num_rows); + ReverseEndian( header.num_cols); + if (header.magic_number != 0x00000803 || header.num_items <= 0) { - cerr << "Error MkMNIST_parseLabels : image-file format error" << endl; + cerr << "Error ReadImageHeader_MNIST : image-file format error" << endl; return; } - - labels.resize(num_items); - for(size_t i=0; i >::IOParam images, - KL::MkMNIST::INParam expr, + KL::MkCNNCpp::INParam expr, KL::String::INParam path, KL::Float64 scale_min, KL::Float64 scale_max, @@ -143,13 +111,108 @@ FABRIC_EXT_EXPORT void MkMNIST_parseImages( return; mnist_header header; - parseMNISTHeader(ifs, header); + ReadImageHeader_MNIST(ifs, header); images.resize(header.num_items); for (size_t i = 0; i < header.num_items; i++) - parseMNISTImage(ifs, header, scale_min, scale_max, x_padding, y_padding, images[i]); + ReadImageData_MNIST(ifs, header, scale_min, scale_max, x_padding, y_padding, images[i]); } +FABRIC_EXT_EXPORT void ReadLabels_MNIST( + KL::VariableArray::IOParam labels, + KL::MkCNNCpp::INParam expr, + KL::String::INParam path) +{ + ifstream ifs(path.data(), ios::in | ios::binary); + if (ifs.bad() || ifs.fail()) + { + cerr << "Error ReadLabels_MkCNNCpp : image-file error" << endl; + return; + } + + uint32_t magic_number; + uint32_t num_items; + ifs.read((char*) &magic_number, 4); + ifs.read((char*) &num_items, 4); + ReverseEndian(magic_number); + ReverseEndian(num_items); + if (magic_number != 0x00000801 || num_items <= 0) + { + cerr << "Error ReadLabels_MkCNNCpp : image-file format error" << endl; + return; + } + + labels.resize(num_items); + for(size_t i=0; i::INParam rgb, + KL::VariableArray >::IOParam images, + KL::VariableArray::IOParam labels) +{ + ifstream file (path.data(), ios::binary); + if (file.is_open()) + { + int n_rows = 32; + int n_cols = 32; + int number_of_images = 10000; + + images.resize(number_of_images); + labels.resize(number_of_images); + + for(int i = 0; i < number_of_images; ++i) + { + // Get the label first + unsigned char tplabel = 0; + file.read((char*) &tplabel, sizeof(tplabel)); + labels[i] = KL::UInt32(tplabel); + + // The the image + KL::VariableArray > channels; + channels.resize(3); + channels[0].resize(n_rows*n_cols); + channels[1].resize(n_rows*n_cols); + channels[2].resize(n_rows*n_cols); + + for(int ch = 0; ch < 3; ++ch) + { + for(int p = 0; p < n_rows*n_cols; ++p) + { + unsigned char temp = 0; + file.read((char*) &temp, sizeof(temp)); + channels[ch][p] = KL::Float64(temp); + } + } + + if(!rgb) + { + images[i].resize(n_rows*n_cols*1); + for(int p = 0; p < n_rows*n_cols; ++p) + images[i][p] = (channels[0][p] + channels[1][p] + channels[2][p])/3.0; + } + else + { + images[i].resize(n_rows*n_cols*3); + for(int p = 0; p < n_rows*n_cols; ++p) + { + images[i][3*p + 0] = channels[0][p]; + images[i][3*p + 1] = channels[1][p]; + images[i][3*p + 2] = channels[2][p]; + } + + } + } + } +} + /********/ template inline @@ -277,6 +340,6 @@ FABRIC_EXT_EXPORT void CurrentDateTime(KL::Traits< KL::String >::IOParam str) { struct tm tstruct; char buf[80]; tstruct = *localtime(&now); - strftime(buf, sizeof(buf), "%Y-%m-%d_%H-%M-%S", &tstruct); + strftime(buf, sizeof(buf), "%Y_%m_%d-%H_%M_%S", &tstruct); str = KL::String(string(buf).c_str()); } diff --git a/core/cnn/c++/MkCNNCpp.fpm.json b/core/cnn/c++/MkCNNCpp.fpm.json new file mode 100644 index 0000000..516bd73 --- /dev/null +++ b/core/cnn/c++/MkCNNCpp.fpm.json @@ -0,0 +1,4 @@ +{ + "libs": "MkCNNCpp", + "code": ["MkCNNCpp.kl" ] +} diff --git a/core/c++/MNIST/MkMNIST.kl b/core/cnn/c++/MkCNNCpp.kl similarity index 85% rename from core/c++/MNIST/MkMNIST.kl rename to core/cnn/c++/MkCNNCpp.kl index 3bcff52..97bef0c 100644 --- a/core/c++/MNIST/MkMNIST.kl +++ b/core/cnn/c++/MkCNNCpp.kl @@ -8,19 +8,16 @@ /* */ /**************************************************************************************************/ -object MkMNIST { + +object MkCNNCpp { Data handle; }; -function Index[] MkMNIST.parseLabels(String path) = "MkMNIST_parseLabels"; +function Index[] MkCNNCpp.readLabels_MNIST(String path) = "ReadLabels_MNIST"; -function Float64[][] MkMNIST.parseImages( - String path, - Float64 scale_min, - Float64 scale_max, - Float64 x_padding, - Float64 y_padding) -= "MkMNIST_parseImages"; +function Float64[][] MkCNNCpp.readImages_MNIST(String path, Float64 scale_min, Float64 scale_max, Float64 x_padding, Float64 y_padding) = "ReadImages_MNIST"; + +function MkCNNCpp.readBatch_CIFAR(String path, Boolean rgb, io Float64 images[][], io UInt32 labels[]) = "ReadBatch_CIFAR"; /******/ diff --git a/core/c++/MNIST/SConstruct b/core/cnn/c++/SConstruct similarity index 96% rename from core/c++/MNIST/SConstruct rename to core/cnn/c++/SConstruct index a9f6830..27c3ee3 100644 --- a/core/c++/MNIST/SConstruct +++ b/core/cnn/c++/SConstruct @@ -30,9 +30,9 @@ flags['CPPFLAGS'] = ['/O2'] fabricBuildEnv.MergeFlags(flags) fabricBuildEnv.Extension( - 'MkMNIST', + 'MkCNNCpp', [ - 'MkMNIST.cpp', 'MkMNIST.kl' + 'MkCNNCpp.cpp', 'MkCNNCpp.kl' ]) diff --git a/core/kl/cnn/MkCNNConfig.kl b/core/kl/cnn/MkCNNConfig.kl deleted file mode 100644 index 82e1b7f..0000000 --- a/core/kl/cnn/MkCNNConfig.kl +++ /dev/null @@ -1,721 +0,0 @@ -/**************************************************************************************************/ -/* */ -/* Informations : */ -/* This code is part of the project MLKL */ -/* */ -/* Contacts : */ -/* couet.julien@gmail.com */ -/* */ -/**************************************************************************************************/ - -require FileIO, Util; -require MLKL; - -/** - The AlembicArchiveReader is a wrapper for the AlembicIArchive. - It provides access to the higher level reader objects such as the AlembicXformReader. - \example - - require MLKL; - - operator entry() {} - - \endexample -*/ - -/**************************************************************************************************/ -/* Activation functions */ -/// Parse the layers configuration parameter relative to the activation function -inline Boolean ParseNeuron( - String line, - io Index neuron_func) -{ - if(line == "id") { - neuron_func = MK_NEURON_IDENTITY; - return true; - } - else if(line == "sig") { - neuron_func = MK_NEURON_SIGMOID; - return true; - } - else if(line == "relu") { - neuron_func = MK_NEURON_RECTIFIEDLINEAR; - return true; - } - else if(line == "tanh") { - neuron_func = MK_NEURON_TANH; - return true; - } - else return false; -} - -/// Parse a string input -inline String ParseStr(String template, String line) { - return line.subString(line.find(template) + String(template).length(), line.length()); -} - -/// Parse a interger input -inline Index ParseInt(String template, String line) { - String str = ParseStr(template, line); - return Index(str.toInteger()); -} - -/// Parse a scalar input -inline Float32 ParseScalar(String template, String line) { - String str = ParseStr(template, line); - return Float32(str.toScalar()); -} - -// Struct to temporary hold the layers' parameters -struct MkCNNLayerParams { - String name; - Float32 epsW; - Float32 epsB; - Float32 momW; - Float32 momB; - Float32 wc; -}; - -/// Parse the layers configuration and create them -inline Boolean ParseLayerPooling( - String layer_name, - io TextReader reader, - io MkCNNLayerInterface layers[]) -{ - Index pool_type, neuron_func, in_size; - Index in_channels, pooling_size, params_counter = 0; - Float32 init_w, init_b; - - String line = reader.readLine(); - if(line.find("pool=") > -1) { - String str = ParseStr("pool=", line); - if(str == "avg") { - pool_type = 0; - params_counter ++; - } - else if(str == "max") { - pool_type = 1; - params_counter ++; - } - line = reader.readLine(); - } - if(line.find("neuron=") > -1) { - String str = ParseStr("neuron=", line); - if(ParseNeuron(str, neuron_func)) - params_counter ++; - line = reader.readLine(); - } - if(line.find("filters=") > -1) { - in_size = ParseInt("filters=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("inChannels=") > -1) { - in_channels = ParseInt("inChannels=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("poolingSize=") > -1) { - pooling_size = ParseInt("poolingSize=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("initW=") > -1) { - init_w = ParseScalar("initW=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("initB=") > -1) { - init_b = ParseScalar("initB=", line); - params_counter ++; line = reader.readLine(); - } - - report("\n--- Pooling " + layer_name + " ---"); - if(params_counter != 7) { - report("Error, wrong parameter order"); - return false; - } - else { - report("pool : " + pool_type); - report("neuron : " + neuron_func); - report("inSize : " + in_size); - report("inChannels : " + in_channels); - report("poolingSize : " + pooling_size); - report("initW : " + init_w); - report("initB : " + init_b); - - if(pool_type == 0) - layers.push(MkCNNLayerAveragePooling( - layer_name, neuron_func,in_size, in_size, - in_channels, pooling_size, init_w, init_b)); - else - layers.push(MkCNNLayerMaxPooling( - layer_name, neuron_func,in_size, in_size, - in_channels, pooling_size, init_w, init_b)); - - return true; - } -} - -/// Parse the layers configuration and create them -inline Boolean ParseLayerConvolutional( - String layer_name, - MkCNNLayerParams layer_params, - io TextReader reader, - io MkCNNLayerInterface layers[]) -{ - Index neuron_func, in_size, window_size; - Index in_channels, out_channels, params_counter = 0; - Float32 init_w, init_b; - - String line = reader.readLine(); - if(line.find("neuron=") > -1) { - String str = ParseStr("neuron=", line); - if(ParseNeuron(str, neuron_func)) - params_counter ++; - line = reader.readLine(); - } - if(line.find("filters=") > -1) { - in_size = ParseInt("filters=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("filterSize=") > -1) { - window_size = ParseInt("filterSize=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("inChannels=") > -1) { - in_channels = ParseInt("inChannels=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("outChannels=") > -1) { - out_channels = ParseInt("outChannels=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("initW=") > -1) { - init_w = ParseScalar("initW=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("initB=") > -1) { - init_b = ParseScalar("initB=", line); - params_counter ++; line = reader.readLine(); - } - - report("\n--- Convolutional " + layer_name + " ---"); - if(params_counter != 7) { - report("Error, wrong parameter order"); - return false; - } - else { - String space; - Index max_length = 8; - String neuron_func_str = String(neuron_func); - String in_size_str = String(in_size); - String window_size_str = String(window_size); - String in_channels_str = String(in_channels); - String out_channels_str = String(out_channels); - - neuron_func_str += space.whiteSpace(max_length - neuron_func_str.length()); - in_size_str += space.whiteSpace(max_length - in_size_str.length()); - window_size_str += space.whiteSpace(max_length - window_size_str.length()); - in_channels_str += space.whiteSpace(max_length - in_channels_str.length()); - out_channels_str += space.whiteSpace(max_length - out_channels_str.length()); - - report("neuron : " + neuron_func_str + "epsW : " + layer_params.epsW); - report("inSize : " + in_size_str + "epsB : " + layer_params.epsB); - report("windowSize : " + window_size_str + "momW : " + layer_params.momW); - report("inChannels : " + in_channels_str + "momB : " + layer_params.momB); - report("outChannels : " + out_channels_str + "wc : " + layer_params.wc); - report("initW : " + init_w); - report("initB : " + init_b); - - layers.push(MkCNNLayerConvolutional( - layer_name, neuron_func, - in_size, in_size, window_size, - in_channels, out_channels, init_w, init_b)); - - return true; - } -} - -/// Parse the layers configuration and create them -inline Boolean ParseLayerFully( - String layer_name, - MkCNNLayerParams layer_params, - io TextReader reader, - io MkCNNLayerInterface layers[]) -{ - Index neuron_func, in_channels, out_channels, params_counter = 0; - Float32 init_w, init_b; - - String line = reader.readLine(); - if(line.find("neuron=") > -1) { - String str = ParseStr("neuron=", line); - if(ParseNeuron(str, neuron_func)) - params_counter ++; - line = reader.readLine(); - } - if(line.find("inChannels=") > -1) { - in_channels = ParseInt("inChannels=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("outChannels=") > -1) { - out_channels = ParseInt("outChannels=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("initW=") > -1) { - init_w = ParseScalar("initW=", line); - params_counter ++; line = reader.readLine(); - } - if(line.find("initB=") > -1) { - init_b = ParseScalar("initB=", line); - params_counter ++; line = reader.readLine(); - } - - report("\n--- Fully-Connected " + layer_name + " ---"); - if(params_counter != 5) { - report("Error, wrong parameter order"); - return false; - } - else { - - String space; - Index max_length = 8; - String neuron_func_str = String(neuron_func); - String in_channels_str = String(in_channels); - String out_channels_str = String(out_channels); - String init_w_str = String(init_w); - String init_b_str = String(init_b); - - neuron_func_str += space.whiteSpace(max_length - neuron_func_str.length()); - in_channels_str += space.whiteSpace(max_length - in_channels_str.length()); - out_channels_str += space.whiteSpace(max_length - out_channels_str.length()); - init_w_str += space.whiteSpace(max_length - init_w_str.length()); - init_b_str += space.whiteSpace(max_length - init_b_str.length()); - - report("neuron : " + neuron_func_str + "epsW : " + layer_params.epsW); - report("inChannels : " + in_channels_str + "epsB : " + layer_params.epsB); - report("outChannels : " + out_channels_str + "momW : " + layer_params.momW); - report("initW : " + init_w_str + "momB : " + layer_params.momB); - report("initB : " + init_b_str + "wc : " + layer_params.wc); - - layers.push(MkCNNLayerFully(layer_name, neuron_func, - in_channels, out_channels, init_w, init_b)); - - return true; - } -} - -/// TO-complete, uses the parameters for the optimization -/// Parse the layers configuration and create them -inline Boolean ParseLayersDefs( - String path_def, - MkCNNLayerParams layers_params[], - io MkCNNLayerInterface layers[]) -{ - TextReader reader(); - if(!reader.open(path_def)) - return false; - - while(!reader.eof()) - { - String line = reader.readLine(); - - // Pass comment - if(line[0] == "#") - line = reader.readLine(); - - // If we hit a new layer - if(line[0] == "[") - { - String layer_name = line; - String layer_type = reader.readLine(); - - MkCNNLayerParams layer_params; - for(Index p=0; p 0) && line[0] != "#") - { - report(line); - - if(line.find("worker=") > -1) { - this.worker = ParseInt("worker=", line); - params_counter ++; - } - if(line.find("gpu=") > -1) { - this.gpu = ParseInt("gpu=", line); - params_counter ++; - } - if(line.find("nbEpoch=") > -1) { - this.epoch = ParseInt("nbEpoch=", line); - params_counter ++; - } - if(line.find("batchSize=") > -1) { - this.batch_size = ParseInt("batch_size=", line); - params_counter ++; - } - if(line.find("optimizer=") > -1) { - this.optimizer = ParseInt("optimizer=", line); - params_counter ++; - } - if(line.find("lossFunction=") > -1) { - this.loss_function = ParseInt("lossFunction=", line); - params_counter ++; - } - if(line.find("layersDefsPath=") > -1) { - this.layers_defs_path = ParseStr("layersDefsPath=", line); - params_counter ++; - } - if(line.find("layersParamsPath=") > -1) { - this.layers_params_path = ParseStr("layersParamsPath=", line); - params_counter ++; - } - if(line.find("trainImagesPath=") > -1) { - this.train_images_path = ParseStr("trainImagesPath=", line); - params_counter ++; - } - if(line.find("testImagesPath=") > -1) { - this.test_images_path = ParseStr("testImagesPath=", line); - params_counter ++; - } - if(line.find("trainLabelsPath=") > -1) { - this.train_labels_path = ParseStr("trainLabelsPath=", line); - params_counter ++; - } - if(line.find("testLabelsPath=") > -1) { - this.test_labels_path = ParseStr("testLabelsPath=", line); - params_counter ++; - } - if(line.find("outputDirPath=") > -1) { - this.main_output_dir_path = ParseStr("outputDirPath=", line); - params_counter ++; - } - } - } - - if(params_counter != 13) { - report("Error, wrong parameter order"); - return false; - } - else { - report("worker : " + this.worker); - report("gpu : " + this.gpu); - report("nbEpochs : " + this.epoch); - report("optimizer : " + this.optimizer); - report("batchSize : " + this.batch_size); - report("lossFunction : " + this.loss_function); - report(""); - report("layersDefs : " + this.layers_defs_path); - report("layerParams : " + this.layers_params_path); - report("outputDir : " + this.main_output_dir_path); - report(""); - report("trainImages : " + this.train_images_path); - report("testImages : " + this.test_images_path); - report("trainLabels : " + this.train_labels_path); - report("testLabels : " + this.test_labels_path); - } - - return reader.close(); -} - -/// To-Do -/// Check if the layers inputs and Zoutputs sizes are correct -private Boolean MkCNNConfig.check(io MkCNNLayerInterface layers[]) { - return true; -} - -/// Crate a directory to save the network -private Boolean MkCNNConfig.initSaving!() { - - FileSystem file_system; - if(!file_system.exists(this.main_output_dir_path)) - return false; - - String date_time; - CurrentDateTime(date_time); - - FilePath file_path(this.main_output_dir_path); - file_path.append(FilePath(date_time)); - this.output_dir_path = file_path.string(); - return file_system.createDirectory(file_path); -} - -/// Configure the whole network form file, incuding the layers -public Boolean MkCNNConfig.config!(String config_path, io MkCNNLayerInterface layers[]) { - - report("\n\n\n-------------------- Configuration --------------------"); - report("\n------------ Network ------------\n"); - if(!this.parse(config_path)) - return false; - - report("\n\n------------ Layers ------------"); - MkCNNLayerParams layers_params[]; - if(!ParseLayersParams(this.layers_params_path, layers_params, layers)) return false; - if(!ParseLayersDefs(this.layers_defs_path, layers_params, layers)) return false; - if(!this.check(layers)) return false; - - return this.initSaving(); -} - -/// Save the layer weights -private Boolean MkCNNConfig.save(io TextWriter writer, Ref layer) { - writer.writeLine("\nname=" + layer.name()); - writer.writeLine("mode="+ layer.modeAsStr()); - writer.writeLine("w="+ layer.weights()); - writer.writeLine("b="+ layer.bias()); - return true; -} - -/// Save the current network's layers -public Boolean MkCNNConfig.save(MkCNNLayers layers) { - - FileSystem file_system; - if(!file_system.exists(this.output_dir_path)) - return false; - - FilePath file_path(this.output_dir_path); - file_path.append("res.mlkl"); - - TextWriter writer(); - if(!writer.open(file_path.string())) - return false; - - for(Index l=0; l current_layer = layers.at(l); - if(current_layer) - { - if(!this.save(writer, current_layer)) - return false; - } - else - return false; - } - return writer.close(); -} - -/// Load a layer weigths -private Boolean MkCNNConfig.load( - String name, - io TextReader reader, - io Ref layer) -{ - String line = reader.readLine(); - if(line.find("mode=") > -1) { - line = reader.readLine(); - } - - if(line.find("w=") > -1) { - String data = ParseStr("w=", line); - data = data.subString(1, data.length()-2); - String datas[] = data.split(","); - - Float64 w[]; w.resize(datas.size()); - for(Index i=0; i= 1.0) return; - this.dropout_rate = rate; -} - -/// Return the filter mode -public Index MkCNNDropout.mode() { - return this.mode; -} - -/// Set the context -public Index MkCNNDropout.context() { - return this.context; -} - -/// Return the context -public MkCNNDropout.context!(Index context) { - this.context = context; -} - -/// \Internal -private MkCNNDropout.shuffle!() { - for(Index i=0; i MkCNNNetwork.optimizer() { - return this.optimizer; -} - -/// Add a new layer to the network -public MkCNNNetwork.add!(io MkCNNLayerInterface layer) { - this.layers.add(layer); -} - -/// Add a stack pf new layers to the network -public MkCNNNetwork.add!(io MkCNNLayerInterface layers[]) { - for(Index i=0; i current = this.layers.head(); - while((current = current.next()) != null) - { - // ignore first input layer - Float64 w[] = current.weight(); - Float64 b[] = current.bias(); - Float64 dw[] = current.weightDiff(0); - Float64 db[] = current.biasDiff(0); - - if (w.size() == 0) continue; - - switch (mode) - { - case MK_GRAD_CHECK_ALL: - for (Index i = 0; i < w.size(); i++) - { - if (this.calcDeltaDiff(ins, v, data_size, w, dw, i) > eps) - return false; - } - for (Index i = 0; i < b.size(); i++) - { - if (this.calcDeltaDiff(ins, v, data_size, b, db, i) > eps) - return false; - } - break; - - case MK_GRAD_CHECK_FIRST: - if (this.calcDeltaDiff(ins, v, data_size, w, dw, 0) > eps) - return false; - if (this.calcDeltaDiff(ins, v, data_size, b, db, 0) > eps) - false; - break; - - case MK_GRAD_CHECK_RANDOM: - for (Index i = 0; i < 10; i++) - { - Index index; UniformRand(0, w.size() - 1, index); - if (this.calcDeltaDiff(ins, v, data_size, w, dw, index) > eps) - return false; - } - for (Index i = 0; i < 10; i++) - { - Index index; UniformRand(0, b.size() - 1, index); - if (this.calcDeltaDiff(ins, v, data_size, b, db, index) > eps) - return false; - } - break; - - default: - report("unknown grad-check type"); - //throw nn_error("unknown grad-check type"); - break; - } - } - - return true; - } -*/ - -/// To Remove == set within the function where it's called -private Float64 MkCNNNetwork.targetValueMin() { - return Float64(this.layers.tail().neuron().scale().x); -} - -/// To Remove == set within the function where it's called -private Float64 MkCNNNetwork.targetValueMax() { - return Float64(this.layers.tail().neuron().scale().y); -} - -/// Convert 1D label array to 2D (image) array -private MkCNNNetwork.label2Vector( - Index batch_index, - Index size, - Index t[], - io Float64 vec[][]) -{ - Index out_dim = this.outDim(); - if(size <= 0 || out_dim <= 0) return; - - Float64 temp[]; temp.resize(out_dim); - for (Index i = 0; i < out_dim; i++) - temp[i] = this.targetValueMin(); - - vec.resize(size); - for (Index i = 0; i < size; i++) - { - if(t[batch_index + i] >= out_dim) - return; - vec[i] = temp.clone(); - vec[i][t[batch_index + i]] = this.targetValueMax(); - } -} - -/// Train one batch -private MkCNNNetwork.trainOnce!( - Index batch_index, - Float64 ins[][], - Float64 t[][], - Index size) -{ - Ref opti = this.optimizer; - if(size == 1) - { - Float64 outs[] = this.fprop(ins[0], 0); - this.bprop(outs, t[0], 0); - this.layers.updateWeights(opti, 1, 1); - } - - else - { - Index num_tasks = size < this.defs.taskSize() ? 1 : this.defs.taskSize(); - Index data_per_thread = size / num_tasks; - Index remaining = size; - - for (Index i = 0; i < num_tasks; i++) - { - Index num = (i == (num_tasks - 1)) ? remaining : data_per_thread; - for (Index j = 0; j < num; j++) - { - Float64 outs[] = this.fprop(ins[batch_index + i*num + j], i); - this.bprop(outs, t[i*num + j], i); - } - remaining -= num; - } - - this.layers.updateWeights(opti, num_tasks, size); - } -} - -/// Overload, Train one batch with 1D label array -private MkCNNNetwork.trainOnce!( - Index batch_index, - Float64 ins[][], - Index t[], - Index size) -{ - Float64 v[][]; - this.label2Vector(batch_index, size, t, v); - this.trainOnce(batch_index, ins, v, size); -} - -private Boolean MkCNNNetwork.isCanonicalLink( - Ref h, - Ref e) -{ - if (h.mode() == MK_NEURON_SIGMOID && e.mode() == MK_LOSS_CE) - return true; - - if (h.mode() == MK_NEURON_TANH && e.mode() == MK_LOSS_CE) - return true; - - if (h.mode() == MK_NEURON_IDENTITY && e.mode() == MK_LOSS_MSE) - return true; - - return false; -} - -/// Computation of the hessian -private MkCNNNetwork.calcHessian!(Float64 ins[][], Index size_init_hessian) { - Index size = Math_min(ins.size(), size_init_hessian); - for(Index i=0; i h = this.layers.tail().neuron(); - if (this.isCanonicalLink(h, this.loss_function)) - { - for (Index i = 0; i < this.outDim(); i++) - delta[i] = outs[i] - t[i]; - } - else - { - for (Index i = 0; i < this.outDim(); i++) - delta[i] = this.loss_function.df(outs[i], t[i]) * h.df(outs[i]); - } - - this.layers.tail().bprop(delta, idx); -} - -/// 2nd Backward propagation, use for Hessian computation -private MkCNNNetwork.bprop2nd!(Float64 outs[]) { - Float64 delta[]; delta.resize(this.outDim()); - - Ref h = this.layers.tail().neuron(); - if (this.isCanonicalLink(h, this.loss_function)) - { - for (Index i = 0; i < this.outDim(); i++) - delta[i] = this.targetValueMax() * h.df(outs[i]); - } - else - { - for (Index i = 0; i < this.outDim(); i++) - delta[i] = this.targetValueMax() * h.df(outs[i]) * h.df(outs[i]); // FIXME - } - - this.layers.tail().bprop2nd(delta); -} diff --git a/core/kl/cnn/MkCNNOptimizer.kl b/core/kl/cnn/MkCNNOptimizer.kl deleted file mode 100644 index 84b54f2..0000000 --- a/core/kl/cnn/MkCNNOptimizer.kl +++ /dev/null @@ -1,209 +0,0 @@ -/**************************************************************************************************/ -/* */ -/* Informations : */ -/* This code is part of the project MLKL */ -/* */ -/* Contacts : */ -/* couet.julien@gmail.com */ -/* */ -/**************************************************************************************************/ - -require MLKL; - - -/**************************************************************************************************/ -/* 2. ASM Batches */ -const Index MK_OPTIMIZER_GD = 0; -const Index MK_OPTIMIZER_GDLM = 1; - -interface MkCNNOptimizerInterface { - update!(Float64 dw[], Float64 H[], io Float64 W[]); - Boolean requiresHessian(); - learningRate!(Float64 learning_rate); - weigthDecay!(Float64 weigth_decay); - Float64 learningRate(); - Float64 weigthDecay(); - display(); - reset!(); -}; - -object MkCNNOptimizerBase : MkCNNOptimizerInterface { - protected Float64 learning_rate; // Learning rate - protected Float64 weigth_decay; // Weight decay -}; - -public MkCNNOptimizerBase.display() { - report("learning_rate " + this.learning_rate); - report("weigthDecay " + this.weigth_decay); -} - -public MkCNNOptimizerBase.update!( - Float64 dw[], - Float64 H[], - io Float64 w[]) {} - -public Boolean MkCNNOptimizerBase.requiresHessian() { - return false; -} - -protected MkCNNOptimizerBase.init!(Float64 learning_rate, Float64 weigth_decay) { - this.learning_rate = learning_rate; - this.weigth_decay = weigth_decay; -} - -public MkCNNOptimizerBase.learningRate!(Float64 learning_rate) { - this.learning_rate = learning_rate; -} - -public MkCNNOptimizerBase.weigthDecay!(Float64 weigth_decay) { - this.weigth_decay = weigth_decay; -} - -public Float64 MkCNNOptimizerBase.learningRate() { - return this.learning_rate; -} - -public Float64 MkCNNOptimizerBase.weigthDecay() { - return this.weigth_decay; -} - -public MkCNNOptimizerBase.reset!() { -} -/* end Constructor/Destructor */ -/**************************************************************************************************/ - - /***********************/ - -/**************************************************************************************************/ -/* 2. ASM Batches */ -object MkCNNOptimizerGD : MkCNNOptimizerBase {}; - -public MkCNNOptimizerGD() { - this.init(0.01, 0.02); -} - -public MkCNNOptimizerGD(Float64 learning_rate, Float64 weigth_decay) { - this.init(learning_rate, weigth_decay); -} - -operator MkCNNOptimizerGDUpdate_task<<>>( - MkCNNOptimizerGD gdlm, - Float64 dw[], - io Float64 w[]) -{ - w[i] -= (gdlm.learningRate() / (w[i] + gdlm.weigthDecay())) * (dw[i]); -} - -public MkCNNOptimizerGD.update!( - Float64 dw[], - Float64 H[], - io Float64 w[]) -{ - MkCNNOptimizerGDUpdate_task<<>>(this, dw, w); -} - -public Boolean MkCNNOptimizerGD.requiresHessian() { - return false; -} -/* end Constructor/Destructor */ -/**************************************************************************************************/ - - /***********************/ - -/**************************************************************************************************/ -/* 2. ASM Batches */ -object MkCNNOptimizerGDLM : MkCNNOptimizerBase {}; - -public MkCNNOptimizerGDLM() { - this.init(0.00085, 0.02); -} - -public MkCNNOptimizerGDLM(Float64 learning_rate, Float64 weigth_decay) { - this.init(learning_rate, weigth_decay); -} - -operator MkCNNOptimizerGDLMUpdate_task<<>>( - Ref gdlm, - Float64 dw[], - Float64 H[], - io Float64 w[]) -{ - w[i] = w[i] - (gdlm.learningRate() / (H[i] + gdlm.weigthDecay())) * (dw[i]); -} - -public MkCNNOptimizerGDLM.update!( - Float64 dw[], - Float64 H[], - io Float64 w[]) -{ - MkCNNOptimizerGDLMUpdate_task<<>>(this, dw, H, w); -} - -public MkCNNOptimizerGDLM.display() { - //report("learning_rate GDLM " + this.learning_rate); - //report("weigthDecay GDLM " + this.weigth_decay); -} - -public Boolean MkCNNOptimizerGDLM.requiresHessian() { - return true; -} -/* end Constructor/Destructor */ -/**************************************************************************************************/ - - /***********************/ - -/**************************************************************************************************/ -/* 2. ASM Batches */ -// http://xcorr.net/2014/01/23/adagrad-eliminating-learning-rates-in-stochastic-gradient-descent/ -/* -object MkCNNOptimizerADAGRAD : MkCNNOptimizerInterface { - Float64 learning_rate; // learning rate - Float64 E[MkHashTable][]; -}; - -public MkCNNOptimizerADAGRAD() { - - this.learning_rate = 0.01; -} - -public MkCNNOptimizerADAGRAD(Float64 learning_rate) { - - this.learning_rate = learning_rate; -} - -operator MkCNNOptimizerADAGRADUpdate_task<<>>( - io MkCNNOptimizerADAGRAD gdlm, - Float64 dw[], - io Float64 w[]) -{ - gdlm.E[i] += dw[i] * dw[i]; - //w[i] -= gdlm.learning_rate * dw[i] / sqrt(gdlm.E[i]); -} - -public MkCNNOptimizerADAGRAD.update( - Float64 dw[], - io Float64 w[]) -{ - if(!E.has(HashTable(w))); - { - - } - if(this.E[w].empty()) - this.E[w] = ; - - MkCNNOptimizerADAGRADUpdate_task<<>>(this, dw, w); -} - -public MkCNNOptimizerADAGRAD.reset() { - - this.E.clear(); -} - -public Boolean MkCNNOptimizerADAGRAD.requiresHessian() { - - return false; -} -*/ -/* end Constructor/Destructor */ -/**************************************************************************************************/ - \ No newline at end of file diff --git a/core/kl/cnn/MkCNNUtils.kl b/core/kl/cnn/MkCNNUtils.kl deleted file mode 100644 index e7f2a23..0000000 --- a/core/kl/cnn/MkCNNUtils.kl +++ /dev/null @@ -1,340 +0,0 @@ -/**************************************************************************************************/ -/* */ -/* Informations : */ -/* This code is part of the project MLKL */ -/* */ -/* Contacts : */ -/* couet.julien@gmail.com */ -/* */ -/**************************************************************************************************/ - -require Math; -require MLKL; - - -/**************************************************************************************************/ -/* Config */ -struct MkCCNDefs { - Index task_size; - Boolean gpu; -}; - -function MkCCNDefs() { - this.task_size = 1; - this.gpu = true; -} - -function Index MkCCNDefs.taskSize() { - return this.task_size; -} - -function Boolean MkCCNDefs.gpu() { - return this.gpu; -} - -function Index MaxIndex(Float64 array[]) { - Float64 max_val = 0; - Index max_index = 0; - for(Index i=0; i > confusion_matrix; -}; - -function MkCNNNetworkResult(Index num_success, Index num_total) { - this.num_success = num_success; - this.num_total = num_total; -} - -function MkCNNNetworkResult() { - this.num_success = 0; - this.num_total = 0; -} - -function Float64 MkCNNNetworkResult.accuracy() { - return this.num_success * 100.0 / this.num_total; -} - -function MkCNNNetworkResult.printSummary() { - report("accuracy : " + this.accuracy() + "% (" + this.num_success + "/" + this.num_total); -} - -function MkCNNNetworkResult.printDetail() { - this.printSummary(); - /* - auto all_labels = labels(); - - os << std::setw(5) << "*" << " "; - for (auto c : all_labels) - os << std::setw(5) << c << " "; - os << std::endl; - - for (auto r : all_labels) { - os << std::setw(5) << r << " "; - for (auto c : all_labels) - os << std::setw(5) << confusion_matrix[r][c] << " "; - os << std::endl; - } - } - - std::set labels() const { - std::set all_labels; - for (auto r : confusion_matrix) { - all_labels.insert(r.first); - for (auto c : r.second) - all_labels.insert(c.first); - } - return all_labels; - } - */ -} - - -// Display iteration info -struct MkEnumerateData { - Index training_size; - Index current_size; - Index batch_size; -}; - -function MkEnumerateData(Index training_size, Index batch_size) { - this.training_size = training_size; - this.batch_size = batch_size; - this.current_size = 0; -} - -function MkEnumerateData.reset!() { - this.current_size = 0; - this.display(); -} - -function MkEnumerateData.display() { - Float32 percent = 100.0 * this.batch_size * Float32(this.current_size) / Float32(this.training_size); - ReportR("Train : " + Index(percent) + "%"); -} - -function MkEnumerateData.update!() { - this.current_size++; - this.display(); -} - - -// Display epoch info -struct MkEnumerateEpoch { - UInt64 start; // To compute the procesing time - Float64 decay_learning_rate; // Update the optimizer learning rate -}; - -function MkEnumerateEpoch() { - this.start = getCurrentTicks(); - this.decay_learning_rate = 0.85; -} - -function MkEnumerateEpoch(Float64 decay_learning_rate) { - this.start = getCurrentTicks(); - this.decay_learning_rate = decay_learning_rate; -} - -/// Display the epoch info and perfor a test -function MkEnumerateEpoch.display( - io Ref nn, - Float64 test_images[][], - Index test_labels[]) -{ - report("Train : 100%"); - Float32 tim = Float32(getSecondsBetweenTicks(this.start, getCurrentTicks())/60.0); - report("Time : " + tim + " mins"); - - MkCNNNetworkResult res = nn.test(test_images, test_labels); - Ref opti = nn.optimizer(); - report("Learning rate : " + Float32(opti.learningRate())); - Float32 succes = 100.0*Float32(res.num_success)/Float32(res.num_total); - report("Test : " + succes + "% succes"); -} - -/// Update the epoch info + optimizer -function MkEnumerateEpoch.update!( - io Ref nn, - Float64 test_images[][], - Index test_labels[]) -{ - // Display the info - this.display(nn, test_images, test_labels); - - // Update the optimization learning rate - Ref opti = nn.optimizer(); - opti.learningRate(opti.learningRate()*this.decay_learning_rate); - opti.learningRate(Math_max(0.00001, opti.learningRate())); - this.start = getCurrentTicks(); -} -/* Outputs */ -/**************************************************************************************************/ \ No newline at end of file diff --git a/core/kl/svm/MkSVM.kl b/core/svm/MkSVM.kl similarity index 100% rename from core/kl/svm/MkSVM.kl rename to core/svm/MkSVM.kl diff --git a/core/kl/svm/MkSVMKernel.kl b/core/svm/MkSVMKernel.kl similarity index 100% rename from core/kl/svm/MkSVMKernel.kl rename to core/svm/MkSVMKernel.kl diff --git a/core/kl/svm/MkSVMMultiClass.kl b/core/svm/MkSVMMultiClass.kl similarity index 100% rename from core/kl/svm/MkSVMMultiClass.kl rename to core/svm/MkSVMMultiClass.kl diff --git a/core/kl/svm/MkSVMSMO.kl b/core/svm/MkSVMSMO.kl similarity index 100% rename from core/kl/svm/MkSVMSMO.kl rename to core/svm/MkSVMSMO.kl diff --git a/resources/mnist/4.bmp b/resources/mnist/4.bmp new file mode 100644 index 0000000..0243daa Binary files /dev/null and b/resources/mnist/4.bmp differ diff --git a/resources/mnist/t10k-images.idx3-ubyte b/resources/mnist/t10k-images.idx3-ubyte new file mode 100644 index 0000000..1170b2c Binary files /dev/null and b/resources/mnist/t10k-images.idx3-ubyte differ diff --git a/resources/mnist/t10k-labels.idx1-ubyte b/resources/mnist/t10k-labels.idx1-ubyte new file mode 100644 index 0000000..d1c3a97 Binary files /dev/null and b/resources/mnist/t10k-labels.idx1-ubyte differ diff --git a/resources/mnist/train-images.idx3-ubyte b/resources/mnist/train-images.idx3-ubyte new file mode 100644 index 0000000..bbce276 Binary files /dev/null and b/resources/mnist/train-images.idx3-ubyte differ diff --git a/resources/mnist/train-labels.idx1-ubyte b/resources/mnist/train-labels.idx1-ubyte new file mode 100644 index 0000000..d6b4c5d Binary files /dev/null and b/resources/mnist/train-labels.idx1-ubyte differ