InputOutput.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:b8dbd15b0c064360840fb4641b3697412fcce88c7b3393966742fa9a056db5e6"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Input/Output data files"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Alexandre M. S. on May 3rd, 2014"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.datasets import load_iris\n",
      "\n",
      "iris = load_iris()\n",
      "\n",
      "X, y = iris.data, iris.target"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print('X.shape = {0}'.format(X.shape))\n",
      "print('y.shape = {0}'.format(y.shape))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "X.shape = (150, 4)\n",
        "y.shape = (150,)\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "##Python object persistence\n",
      "The [`shelve`](https://docs.python.org/3.4/library/shelve.html) module allows to store arbitrary Python objects (anything that the [`pickle`](https://docs.python.org/3.4/library/pickle.html#module-pickle) module can handle) into a file."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import shelve\n",
      "db = shelve.open('iris_data.shelf')\n",
      "db['X'] = X\n",
      "db['y'] = y\n",
      "db.close()\n",
      "\n",
      "dbr = shelve.open('iris_data.shelf')\n",
      "dbr.keys()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "['y', 'X']"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "##Numpy text file IO"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "\n",
      "np.savetxt?"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.savetxt('iris_data.txt', X, fmt='%.4e')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!more iris_data.txt"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "5.1000e+00 3.5000e+00 1.4000e+00 2.0000e-01\r\n",
        "4.9000e+00 3.0000e+00 1.4000e+00 2.0000e-01\r\n",
        "4.7000e+00 3.2000e+00 1.3000e+00 2.0000e-01\r\n",
        "4.6000e+00 3.1000e+00 1.5000e+00 2.0000e-01\r\n",
        "5.0000e+00 3.6000e+00 1.4000e+00 2.0000e-01\r\n",
        "5.4000e+00 3.9000e+00 1.7000e+00 4.0000e-01\r\n",
        "4.6000e+00 3.4000e+00 1.4000e+00 3.0000e-01\r\n",
        "5.0000e+00 3.4000e+00 1.5000e+00 2.0000e-01\r\n",
        "4.4000e+00 2.9000e+00 1.4000e+00 2.0000e-01\r\n",
        "4.9000e+00 3.1000e+00 1.5000e+00 1.0000e-01\r\n",
        "5.4000e+00 3.7000e+00 1.5000e+00 2.0000e-01\r\n",
        "4.8000e+00 3.4000e+00 1.6000e+00 2.0000e-01\r\n",
        "4.8000e+00 3.0000e+00 1.4000e+00 1.0000e-01\r\n",
        "4.3000e+00 3.0000e+00 1.1000e+00 1.0000e-01\r\n",
        "5.8000e+00 4.0000e+00 1.2000e+00 2.0000e-01\r\n",
        "5.7000e+00 4.4000e+00 1.5000e+00 4.0000e-01\r\n",
        "5.4000e+00 3.9000e+00 1.3000e+00 4.0000e-01\r\n",
        "5.1000e+00 3.5000e+00 1.4000e+00 3.0000e-01\r\n",
        "5.7000e+00 3.8000e+00 1.7000e+00 3.0000e-01\r\n",
        "5.1000e+00 3.8000e+00 1.5000e+00 3.0000e-01\r\n",
        "5.4000e+00 3.4000e+00 1.7000e+00 2.0000e-01\r\n",
        "5.1000e+00 3.7000e+00 1.5000e+00 4.0000e-01\r\n",
        "4.6000e+00 3.6000e+00 1.0000e+00 2.0000e-01\r\n",
        "5.1000e+00 3.3000e+00 1.7000e+00 5.0000e-01\r\n",
        "4.8000e+00 3.4000e+00 1.9000e+00 2.0000e-01\r\n",
        "5.0000e+00 3.0000e+00 1.6000e+00 2.0000e-01\r\n",
        "5.0000e+00 3.4000e+00 1.6000e+00 4.0000e-01\r\n",
        "5.2000e+00 3.5000e+00 1.5000e+00 2.0000e-01\r\n",
        "5.2000e+00 3.4000e+00 1.4000e+00 2.0000e-01\r\n",
        "4.7000e+00 3.2000e+00 1.6000e+00 2.0000e-01\r\n",
        "4.8000e+00 3.1000e+00 1.6000e+00 2.0000e-01\r\n",
        "5.4000e+00 3.4000e+00 1.5000e+00 4.0000e-01\r\n",
        "5.2000e+00 4.1000e+00 1.5000e+00 1.0000e-01\r\n",
        "5.5000e+00 4.2000e+00 1.4000e+00 2.0000e-01\r\n",
        "4.9000e+00 3.1000e+00 1.5000e+00 1.0000e-01\r\n",
        "5.0000e+00 3.2000e+00 1.2000e+00 2.0000e-01\r\n",
        "5.5000e+00 3.5000e+00 1.3000e+00 2.0000e-01\r\n",
        "4.9000e+00 3.1000e+00 1.5000e+00 1.0000e-01\r\n",
        "4.4000e+00 3.0000e+00 1.3000e+00 2.0000e-01\r\n",
        "5.1000e+00 3.4000e+00 1.5000e+00 2.0000e-01\r\n",
        "5.0000e+00 3.5000e+00 1.3000e+00 3.0000e-01\r\n",
        "4.5000e+00 2.3000e+00 1.3000e+00 3.0000e-01\r\n",
        "4.4000e+00 3.2000e+00 1.3000e+00 2.0000e-01\r\n",
        "5.0000e+00 3.5000e+00 1.6000e+00 6.0000e-01\r\n",
        "5.1000e+00 3.8000e+00 1.9000e+00 4.0000e-01\r\n",
        "4.8000e+00 3.0000e+00 1.4000e+00 3.0000e-01\r\n",
        "5.1000e+00 3.8000e+00 1.6000e+00 2.0000e-01\r\n",
        "4.6000e+00 3.2000e+00 1.4000e+00 2.0000e-01\r\n",
        "5.3000e+00 3.7000e+00 1.5000e+00 2.0000e-01\r\n",
        "5.0000e+00 3.3000e+00 1.4000e+00 2.0000e-01\r\n",
        "7.0000e+00 3.2000e+00 4.7000e+00 1.4000e+00\r\n",
        "6.4000e+00 3.2000e+00 4.5000e+00 1.5000e+00\r\n",
        "6.9000e+00 3.1000e+00 4.9000e+00 1.5000e+00\r\n",
        "5.5000e+00 2.3000e+00 4.0000e+00 1.3000e+00\r\n",
        "6.5000e+00 2.8000e+00 4.6000e+00 1.5000e+00\r\n",
        "5.7000e+00 2.8000e+00 4.5000e+00 1.3000e+00\r\n",
        "\u001b[7m--More--(37%)\u001b[m"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "\u001b[K\r\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.savetxt('iris_data.csv', X, fmt='%.3f', delimiter=',')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!more iris_data.csv"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "5.100,3.500,1.400,0.200\r\n",
        "4.900,3.000,1.400,0.200\r\n",
        "4.700,3.200,1.300,0.200\r\n",
        "4.600,3.100,1.500,0.200\r\n",
        "5.000,3.600,1.400,0.200\r\n",
        "5.400,3.900,1.700,0.400\r\n",
        "4.600,3.400,1.400,0.300\r\n",
        "5.000,3.400,1.500,0.200\r\n",
        "4.400,2.900,1.400,0.200\r\n",
        "4.900,3.100,1.500,0.100\r\n",
        "5.400,3.700,1.500,0.200\r\n",
        "4.800,3.400,1.600,0.200\r\n",
        "4.800,3.000,1.400,0.100\r\n",
        "4.300,3.000,1.100,0.100\r\n",
        "5.800,4.000,1.200,0.200\r\n",
        "5.700,4.400,1.500,0.400\r\n",
        "5.400,3.900,1.300,0.400\r\n",
        "5.100,3.500,1.400,0.300\r\n",
        "5.700,3.800,1.700,0.300\r\n",
        "5.100,3.800,1.500,0.300\r\n",
        "5.400,3.400,1.700,0.200\r\n",
        "5.100,3.700,1.500,0.400\r\n",
        "4.600,3.600,1.000,0.200\r\n",
        "5.100,3.300,1.700,0.500\r\n",
        "4.800,3.400,1.900,0.200\r\n",
        "5.000,3.000,1.600,0.200\r\n",
        "5.000,3.400,1.600,0.400\r\n",
        "5.200,3.500,1.500,0.200\r\n",
        "5.200,3.400,1.400,0.200\r\n",
        "4.700,3.200,1.600,0.200\r\n",
        "4.800,3.100,1.600,0.200\r\n",
        "5.400,3.400,1.500,0.400\r\n",
        "5.200,4.100,1.500,0.100\r\n",
        "5.500,4.200,1.400,0.200\r\n",
        "4.900,3.100,1.500,0.100\r\n",
        "5.000,3.200,1.200,0.200\r\n",
        "5.500,3.500,1.300,0.200\r\n",
        "4.900,3.100,1.500,0.100\r\n",
        "4.400,3.000,1.300,0.200\r\n",
        "5.100,3.400,1.500,0.200\r\n",
        "5.000,3.500,1.300,0.300\r\n",
        "4.500,2.300,1.300,0.300\r\n",
        "4.400,3.200,1.300,0.200\r\n",
        "5.000,3.500,1.600,0.600\r\n",
        "5.100,3.800,1.900,0.400\r\n",
        "4.800,3.000,1.400,0.300\r\n",
        "5.100,3.800,1.600,0.200\r\n",
        "4.600,3.200,1.400,0.200\r\n",
        "5.300,3.700,1.500,0.200\r\n",
        "5.000,3.300,1.400,0.200\r\n",
        "7.000,3.200,4.700,1.400\r\n",
        "6.400,3.200,4.500,1.500\r\n",
        "6.900,3.100,4.900,1.500\r\n",
        "5.500,2.300,4.000,1.300\r\n",
        "6.500,2.800,4.600,1.500\r\n",
        "5.700,2.800,4.500,1.300\r\n",
        "\u001b[7m--More--(37%)\u001b[m"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "\u001b[K\r\n"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "Xread = np.loadtxt('iris_data.csv', delimiter=',')\n",
      "Xread[0,:]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 22,
       "text": [
        "array([ 5.1,  3.5,  1.4,  0.2])"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "##Numpy pickling"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.save('iris_X.npy', X)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "Xr = np.load('iris_X.npy')\n",
      "Xr[0,:]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "array([ 5.1,  3.5,  1.4,  0.2])"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "Xr_slicer = np.load('iris_X.npy', 'r')\n",
      "Xr_slicer"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "memmap([[ 5.1,  3.5,  1.4,  0.2],\n",
        "       [ 4.9,  3. ,  1.4,  0.2],\n",
        "       [ 4.7,  3.2,  1.3,  0.2],\n",
        "       [ 4.6,  3.1,  1.5,  0.2],\n",
        "       [ 5. ,  3.6,  1.4,  0.2],\n",
        "       [ 5.4,  3.9,  1.7,  0.4],\n",
        "       [ 4.6,  3.4,  1.4,  0.3],\n",
        "       [ 5. ,  3.4,  1.5,  0.2],\n",
        "       [ 4.4,  2.9,  1.4,  0.2],\n",
        "       [ 4.9,  3.1,  1.5,  0.1],\n",
        "       [ 5.4,  3.7,  1.5,  0.2],\n",
        "       [ 4.8,  3.4,  1.6,  0.2],\n",
        "       [ 4.8,  3. ,  1.4,  0.1],\n",
        "       [ 4.3,  3. ,  1.1,  0.1],\n",
        "       [ 5.8,  4. ,  1.2,  0.2],\n",
        "       [ 5.7,  4.4,  1.5,  0.4],\n",
        "       [ 5.4,  3.9,  1.3,  0.4],\n",
        "       [ 5.1,  3.5,  1.4,  0.3],\n",
        "       [ 5.7,  3.8,  1.7,  0.3],\n",
        "       [ 5.1,  3.8,  1.5,  0.3],\n",
        "       [ 5.4,  3.4,  1.7,  0.2],\n",
        "       [ 5.1,  3.7,  1.5,  0.4],\n",
        "       [ 4.6,  3.6,  1. ,  0.2],\n",
        "       [ 5.1,  3.3,  1.7,  0.5],\n",
        "       [ 4.8,  3.4,  1.9,  0.2],\n",
        "       [ 5. ,  3. ,  1.6,  0.2],\n",
        "       [ 5. ,  3.4,  1.6,  0.4],\n",
        "       [ 5.2,  3.5,  1.5,  0.2],\n",
        "       [ 5.2,  3.4,  1.4,  0.2],\n",
        "       [ 4.7,  3.2,  1.6,  0.2],\n",
        "       [ 4.8,  3.1,  1.6,  0.2],\n",
        "       [ 5.4,  3.4,  1.5,  0.4],\n",
        "       [ 5.2,  4.1,  1.5,  0.1],\n",
        "       [ 5.5,  4.2,  1.4,  0.2],\n",
        "       [ 4.9,  3.1,  1.5,  0.1],\n",
        "       [ 5. ,  3.2,  1.2,  0.2],\n",
        "       [ 5.5,  3.5,  1.3,  0.2],\n",
        "       [ 4.9,  3.1,  1.5,  0.1],\n",
        "       [ 4.4,  3. ,  1.3,  0.2],\n",
        "       [ 5.1,  3.4,  1.5,  0.2],\n",
        "       [ 5. ,  3.5,  1.3,  0.3],\n",
        "       [ 4.5,  2.3,  1.3,  0.3],\n",
        "       [ 4.4,  3.2,  1.3,  0.2],\n",
        "       [ 5. ,  3.5,  1.6,  0.6],\n",
        "       [ 5.1,  3.8,  1.9,  0.4],\n",
        "       [ 4.8,  3. ,  1.4,  0.3],\n",
        "       [ 5.1,  3.8,  1.6,  0.2],\n",
        "       [ 4.6,  3.2,  1.4,  0.2],\n",
        "       [ 5.3,  3.7,  1.5,  0.2],\n",
        "       [ 5. ,  3.3,  1.4,  0.2],\n",
        "       [ 7. ,  3.2,  4.7,  1.4],\n",
        "       [ 6.4,  3.2,  4.5,  1.5],\n",
        "       [ 6.9,  3.1,  4.9,  1.5],\n",
        "       [ 5.5,  2.3,  4. ,  1.3],\n",
        "       [ 6.5,  2.8,  4.6,  1.5],\n",
        "       [ 5.7,  2.8,  4.5,  1.3],\n",
        "       [ 6.3,  3.3,  4.7,  1.6],\n",
        "       [ 4.9,  2.4,  3.3,  1. ],\n",
        "       [ 6.6,  2.9,  4.6,  1.3],\n",
        "       [ 5.2,  2.7,  3.9,  1.4],\n",
        "       [ 5. ,  2. ,  3.5,  1. ],\n",
        "       [ 5.9,  3. ,  4.2,  1.5],\n",
        "       [ 6. ,  2.2,  4. ,  1. ],\n",
        "       [ 6.1,  2.9,  4.7,  1.4],\n",
        "       [ 5.6,  2.9,  3.6,  1.3],\n",
        "       [ 6.7,  3.1,  4.4,  1.4],\n",
        "       [ 5.6,  3. ,  4.5,  1.5],\n",
        "       [ 5.8,  2.7,  4.1,  1. ],\n",
        "       [ 6.2,  2.2,  4.5,  1.5],\n",
        "       [ 5.6,  2.5,  3.9,  1.1],\n",
        "       [ 5.9,  3.2,  4.8,  1.8],\n",
        "       [ 6.1,  2.8,  4. ,  1.3],\n",
        "       [ 6.3,  2.5,  4.9,  1.5],\n",
        "       [ 6.1,  2.8,  4.7,  1.2],\n",
        "       [ 6.4,  2.9,  4.3,  1.3],\n",
        "       [ 6.6,  3. ,  4.4,  1.4],\n",
        "       [ 6.8,  2.8,  4.8,  1.4],\n",
        "       [ 6.7,  3. ,  5. ,  1.7],\n",
        "       [ 6. ,  2.9,  4.5,  1.5],\n",
        "       [ 5.7,  2.6,  3.5,  1. ],\n",
        "       [ 5.5,  2.4,  3.8,  1.1],\n",
        "       [ 5.5,  2.4,  3.7,  1. ],\n",
        "       [ 5.8,  2.7,  3.9,  1.2],\n",
        "       [ 6. ,  2.7,  5.1,  1.6],\n",
        "       [ 5.4,  3. ,  4.5,  1.5],\n",
        "       [ 6. ,  3.4,  4.5,  1.6],\n",
        "       [ 6.7,  3.1,  4.7,  1.5],\n",
        "       [ 6.3,  2.3,  4.4,  1.3],\n",
        "       [ 5.6,  3. ,  4.1,  1.3],\n",
        "       [ 5.5,  2.5,  4. ,  1.3],\n",
        "       [ 5.5,  2.6,  4.4,  1.2],\n",
        "       [ 6.1,  3. ,  4.6,  1.4],\n",
        "       [ 5.8,  2.6,  4. ,  1.2],\n",
        "       [ 5. ,  2.3,  3.3,  1. ],\n",
        "       [ 5.6,  2.7,  4.2,  1.3],\n",
        "       [ 5.7,  3. ,  4.2,  1.2],\n",
        "       [ 5.7,  2.9,  4.2,  1.3],\n",
        "       [ 6.2,  2.9,  4.3,  1.3],\n",
        "       [ 5.1,  2.5,  3. ,  1.1],\n",
        "       [ 5.7,  2.8,  4.1,  1.3],\n",
        "       [ 6.3,  3.3,  6. ,  2.5],\n",
        "       [ 5.8,  2.7,  5.1,  1.9],\n",
        "       [ 7.1,  3. ,  5.9,  2.1],\n",
        "       [ 6.3,  2.9,  5.6,  1.8],\n",
        "       [ 6.5,  3. ,  5.8,  2.2],\n",
        "       [ 7.6,  3. ,  6.6,  2.1],\n",
        "       [ 4.9,  2.5,  4.5,  1.7],\n",
        "       [ 7.3,  2.9,  6.3,  1.8],\n",
        "       [ 6.7,  2.5,  5.8,  1.8],\n",
        "       [ 7.2,  3.6,  6.1,  2.5],\n",
        "       [ 6.5,  3.2,  5.1,  2. ],\n",
        "       [ 6.4,  2.7,  5.3,  1.9],\n",
        "       [ 6.8,  3. ,  5.5,  2.1],\n",
        "       [ 5.7,  2.5,  5. ,  2. ],\n",
        "       [ 5.8,  2.8,  5.1,  2.4],\n",
        "       [ 6.4,  3.2,  5.3,  2.3],\n",
        "       [ 6.5,  3. ,  5.5,  1.8],\n",
        "       [ 7.7,  3.8,  6.7,  2.2],\n",
        "       [ 7.7,  2.6,  6.9,  2.3],\n",
        "       [ 6. ,  2.2,  5. ,  1.5],\n",
        "       [ 6.9,  3.2,  5.7,  2.3],\n",
        "       [ 5.6,  2.8,  4.9,  2. ],\n",
        "       [ 7.7,  2.8,  6.7,  2. ],\n",
        "       [ 6.3,  2.7,  4.9,  1.8],\n",
        "       [ 6.7,  3.3,  5.7,  2.1],\n",
        "       [ 7.2,  3.2,  6. ,  1.8],\n",
        "       [ 6.2,  2.8,  4.8,  1.8],\n",
        "       [ 6.1,  3. ,  4.9,  1.8],\n",
        "       [ 6.4,  2.8,  5.6,  2.1],\n",
        "       [ 7.2,  3. ,  5.8,  1.6],\n",
        "       [ 7.4,  2.8,  6.1,  1.9],\n",
        "       [ 7.9,  3.8,  6.4,  2. ],\n",
        "       [ 6.4,  2.8,  5.6,  2.2],\n",
        "       [ 6.3,  2.8,  5.1,  1.5],\n",
        "       [ 6.1,  2.6,  5.6,  1.4],\n",
        "       [ 7.7,  3. ,  6.1,  2.3],\n",
        "       [ 6.3,  3.4,  5.6,  2.4],\n",
        "       [ 6.4,  3.1,  5.5,  1.8],\n",
        "       [ 6. ,  3. ,  4.8,  1.8],\n",
        "       [ 6.9,  3.1,  5.4,  2.1],\n",
        "       [ 6.7,  3.1,  5.6,  2.4],\n",
        "       [ 6.9,  3.1,  5.1,  2.3],\n",
        "       [ 5.8,  2.7,  5.1,  1.9],\n",
        "       [ 6.8,  3.2,  5.9,  2.3],\n",
        "       [ 6.7,  3.3,  5.7,  2.5],\n",
        "       [ 6.7,  3. ,  5.2,  2.3],\n",
        "       [ 6.3,  2.5,  5. ,  1.9],\n",
        "       [ 6.5,  3. ,  5.2,  2. ],\n",
        "       [ 6.2,  3.4,  5.4,  2.3],\n",
        "       [ 5.9,  3. ,  5.1,  1.8]])"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.memmap?"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.savez('all_iris_data.npz', Xiris=X, yiris=y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "iris_npz_content = np.load('all_iris_data.npz')\n",
      "iris_npz_content.files"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "['Xiris', 'yiris']"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "iris_npz_content['Xiris'][0,:]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 15,
       "text": [
        "array([ 5.1,  3.5,  1.4,  0.2])"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "##Scipy IO\n",
      "Scipy offers a set of methods for accessing other common types of files as Matlab .mat, Matrix Market, IDL, Wave sound files, .arff Weka files and NetCDF. More details [here](http://docs.scipy.org/doc/scipy/reference/tutorial/io.html)."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import scipy.io as sio"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 17
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "###Matlab files"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "\n",
      "vect = np.arange(10)\n",
      "sio.savemat('np_vector.mat', {'vect':vect, 'vect2': vect})"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mat_content = sio.loadmat('np_vector.mat')\n",
      "np.all(vect == mat_content['vect'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 19,
       "text": [
        "True"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "##[HDF5](http://www.hdfgroup.org/HDF5/)\n",
      "HDF5 is a file format with many interesting [advantages](http://www.hdfgroup.org/why_hdf/). It is becoming very popular, to the point that the latest versions of Matlab and IDL use it.\n",
      "\n",
      "There are two ways of using HDF5 files in Python: through [h5py](http://www.h5py.org/) or [pyTables](http://www.pytables.org)."
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "###H5py"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import h5py\n",
      "\n",
      "f = h5py.File(\"mytestfile.hdf5\", \"w\")\n",
      "#with h5py.File(\"mytestfile.hdf5\", \"w\") as f:\n",
      "dset = f.create_dataset(\"mydataset\", (100,), dtype='i')\n",
      "print(dset.shape)\n",
      "print(dset.dtype)\n",
      "dset[...] = np.arange(100)\n",
      "f.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(100,)\n",
        "int32\n"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f = h5py.File(\"mytestfile.hdf5\", \"r\")\n",
      "\n",
      "dset = f['mydataset']\n",
      "print(dset)\n",
      "print(dset[0:100:10])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<HDF5 dataset \"mydataset\": shape (100,), type \"<i4\">\n",
        "[ 0 10 20 30 40 50 60 70 80 90]\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!h5ls -rd mytestfile.hdf5"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "/                        Group\r\n",
        "/mydataset               Dataset {100}\r\n",
        "    Data:\r\n",
        "        (0) 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,\r\n",
        "        (25) 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46,\r\n",
        "        (47) 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68,\r\n",
        "        (69) 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90,\r\n",
        "        (91) 91, 92, 93, 94, 95, 96, 97, 98, 99\r\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "###pyTables"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from tables import *\n",
      "\n",
      "# imagine that we have a particle detector and we want to create \n",
      "#a table object in order to save data retrieved from it.\n",
      "#You need first to define the table, the number of columns it has, \n",
      "#what kind of object is contained in each column, and so on.\n",
      "class Particle(IsDescription):\n",
      "    name      = StringCol(16)   # 16-character String\n",
      "    idnumber  = Int64Col()      # Signed 64-bit integer\n",
      "    ADCcount  = UInt16Col()     # Unsigned short integer\n",
      "    TDCcount  = UInt8Col()      # unsigned byte\n",
      "    grid_i    = Int32Col()      # 32-bit integer\n",
      "    grid_j    = Int32Col()      # 32-bit integer\n",
      "    pressure  = Float32Col()    # float  (single-precision)\n",
      "    energy    = Float64Col()    # double (double-precision)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Use the top-level open_file() function to create a PyTables file:\n",
      "h5file = open_file(\"tutorial1.h5\", mode = \"w\", title = \"Test file\")\n",
      "\n",
      "#to better organize our data, we will create a group called \n",
      "#detector that branches from the root node. \n",
      "#We will save our particle data table in this group:\n",
      "group = h5file.create_group(\"/\", 'detector', 'Detector information')\n",
      "\n",
      "#We create the Table instance under group. We assign this table the \n",
      "#node name \u201creadout\u201d. The Particle class declared before is \n",
      "#the description parameter (to define the columns of the table) \n",
      "#and finally we set \u201cReadout example\u201d as the Table title.\n",
      "#With all this information, a new Table instance is created and\n",
      "#assigned to the variable table.\n",
      "table = h5file.create_table(group, 'readout', Particle, \"Readout example\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#To fill this table with some values. \n",
      "#First we will get a pointer to the Row (see The Row class) \n",
      "#instance of this table instance:\n",
      "\n",
      "#The row attribute of table points to the Row instance that will be \n",
      "#used to write data rows into the table.\n",
      "particle = table.row\n",
      "\n",
      "for i in list(range(10)):\n",
      "    particle['name']  = 'Particle: %6d' % (i)\n",
      "    particle['TDCcount'] = i % 256\n",
      "    particle['ADCcount'] = (i * 256) % (1 << 16)\n",
      "    particle['grid_i'] = i\n",
      "    particle['grid_j'] = 10 - i\n",
      "    particle['pressure'] = float(i*i)\n",
      "    particle['energy'] = float(particle['pressure'] ** 4)\n",
      "    particle['idnumber'] = i * (2 ** 34)\n",
      "    # Insert a new particle record\n",
      "    particle.append()\n",
      "\n",
      "#After we have processed all our data, we should flush \n",
      "#the table\u2019s I/O buffer if we want to write all this data to disk.\n",
      "table.flush()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#reading from the file\n",
      "table = h5file.root.detector.readout\n",
      "pressure = [x['pressure'] for x in table.iterrows() if x['TDCcount'] > 3 and 20 <= x['pressure'] < 50]\n",
      "print(pressure)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[25.0, 36.0, 49.0]\n"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#PyTables do offer other, more powerful ways of performing \n",
      "#selections which may be more suitable if you have very large \n",
      "#tables or if you need very high query speeds. They are called \n",
      "#in-kernel and indexed queries, and you can use them through \n",
      "#Table.where() and other related methods.\n",
      "\n",
      "names = [ x['name'] for x in table.where(\"\"\"(TDCcount > 3) & (20 <= pressure) & (pressure < 50)\"\"\") ]\n",
      "print(names)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[b'Particle:      5', b'Particle:      6', b'Particle:      7']\n"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "h5file.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!h5ls -rd tutorial1.h5"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "/                        Group\r\n",
        "/detector                Group\r\n",
        "/detector/readout        Dataset {10/Inf}\r\n",
        "    Data:\r\n",
        "        (0) {0, 0, 0, 0, 10, 0, \"Particle:      0\", 0},\r\n",
        "        (1) {256, 1, 1, 1, 9, 17179869184, \"Particle:      1\", 1},\r\n",
        "        (2) {512, 2, 256, 2, 8, 34359738368, \"Particle:      2\", 4},\r\n",
        "        (3) {768, 3, 6561, 3, 7, 51539607552, \"Particle:      3\", 9},\r\n",
        "        (4) {1024, 4, 65536, 4, 6, 68719476736, \"Particle:      4\", 16},\r\n",
        "        (5) {1280, 5, 390625, 5, 5, 85899345920, \"Particle:      5\", 25},\r\n",
        "        (6) {1536, 6, 1679616, 6, 4, 103079215104, \"Particle:      6\", 36},\r\n",
        "        (7) {1792, 7, 5764801, 7, 3, 120259084288, \"Particle:      7\", 49},\r\n",
        "        (8) {2048, 8, 16777216, 8, 2, 137438953472, \"Particle:      8\", 64},\r\n",
        "        (9) {2304, 9, 43046721, 9, 1, 154618822656, \"Particle:      9\", 81}\r\n"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "##References\n",
      "\n",
      "+ http://scikit-learn.org\n",
      "+ http://docs.scipy.org/doc/scipy/reference/tutorial/io.html\n",
      "\n",
      "+ http://www.hdfgroup.org/HDF5/\n",
      "+ http://www.hdfgroup.org/why_hdf/\n",
      "\n",
      "+ http://www.h5py.org/\n",
      "+ http://www.pytables.org"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}