project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Intro to Hardware Security and Trust Final Project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV,train_test_split\n",
    "from sklearn.cluster import KMeans,DBSCAN\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import accuracy_score,precision_score,recall_score,confusion_matrix\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv\n",
    "import random\n",
    "import os\n",
    "from statistics import mean\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Splitting Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "xlsx_path = 'C:\\\\Users\\patri\\\\git\\\\HSTFinalProject\\\\Classifier-for-Hardware-Trojan-Detection\\\\ROFreq\\\\ROFreq\\\\Chip'\n",
    "csv_path ='C:\\\\Users\\patri\\\\git\\HSTFinalProject\\Classifier-for-Hardware-Trojan-Detection\\ROFreq_C\\Chip'\n",
    "\n",
    "for i in np.arange(1,34):\n",
    "    data_xls = pd.read_excel('{}{}.xlsx'.format(xlsx_path,i), dtype=str, index_col=None)\n",
    "    data_xls.to_csv('{}{}.csv'.format(csv_path,i), encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def csv_read(chip_num,type,csv_path):\n",
    "    data= []\n",
    "    data_list= []\n",
    "    label_list = np.zeros(25)\n",
    "    label_list[0] = 1\n",
    "    label_list[-1] = 1\n",
    "    with open('{}{}.csv'.format(csv_path,chip_num)) as csvfile:\n",
    "        csv_reader = csv.reader(csvfile)\n",
    "   \n",
    "        for row in csv_reader:\n",
    "            # rounding\n",
    "            row = [round(float(num[0:12]),3) for num in row]     \n",
    "            data_list.append(row)   \n",
    "\n",
    "        if type == 'TI':\n",
    "            data.append(data_list[0])\n",
    "            data.append(data_list[-1])\n",
    "            label = [1,1]\n",
    "        elif type == 'TF':\n",
    "            data = data_list[1:-1]\n",
    "            label = label_list[1:-1]\n",
    "        else:\n",
    "            data = data_list\n",
    "            label = label_list \n",
    "                  \n",
    "        \n",
    "    return data,label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_test_split(num_sel,case_sel,csv_path):\n",
    "    chips_num = np.arange(1,34)\n",
    "    indices = range(33)\n",
    "    # randomly select #num chips\n",
    "    sel = random.sample(indices,num_sel)\n",
    "    # split tain and test data\n",
    "    train_chips = chips_num[sel]\n",
    "    test_chips = np.delete(chips_num,sel)\n",
    "    print('Training chips num: ',train_chips)\n",
    "    temp = []\n",
    "\n",
    "    for i in range(int(num_sel/2)):\n",
    "        temp.extend(['TF','TI'])\n",
    " \n",
    "    type_sel_1 = random.sample(temp,num_sel)\n",
    "    type_sel_3 = random.choices(['TI','TF'],k=num_sel)\n",
    "    \n",
    "    if case_sel == 1:\n",
    "        print('Type selection:', type_sel_1)\n",
    "        print('TI:',type_sel_1.count('TI'))\n",
    "        print('TF:',type_sel_1.count('TF'))\n",
    "\n",
    "    else:\n",
    "        print('Type selection:', type_sel_3)\n",
    "        print('TI:',type_sel_3.count('TI'))\n",
    "        print('TF:',type_sel_3.count('TF'))\n",
    "\n",
    "    cnt = 0\n",
    "    train_data = []\n",
    "    train_label = []\n",
    "\n",
    "    for chip_num in train_chips:\n",
    "        if case_sel == 1:\n",
    "            data,label = csv_read(chip_num,type_sel_1[cnt],csv_path)\n",
    "        elif case_sel ==3:\n",
    "            data,label = csv_read(chip_num,type_sel_3[cnt],csv_path)\n",
    "        train_data.extend(data)\n",
    "        train_label.extend(label)\n",
    "        cnt += 1\n",
    "\n",
    "    test_data = []\n",
    "    test_label = []\n",
    "    for chip_num in test_chips:\n",
    "        data,label = csv_read(chip_num,'ALL',csv_path)\n",
    "        test_data.extend(data)\n",
    "        test_label.extend(label)\n",
    "    \n",
    "    return train_data, train_label, test_data, test_label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  CASE 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Case 1 Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training chips num:  [ 5  9 26  3 10 23  1 20 31  6 13 19 12 33 27 25 11 17  8  2 18  4 14 28]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 3 11  8 18 30 20 33 22  1  2 14 13]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 4  9 20  8 24 19]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n"
     ]
    }
   ],
   "source": [
    "# arg1 => number of training samples\n",
    "# arg2 => number of case (enter 1 or 3 only)\n",
    "train24_data, train24_label, test24_data, test24_label = train_test_split(24,1,csv_path)\n",
    "train12_data, train12_label, test12_data, test12_label = train_test_split(12,1,csv_path)\n",
    "train6_data, train6_label, test6_data, test6_label = train_test_split(6,1,csv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>180.0</td>\n",
       "      <td>191.2</td>\n",
       "      <td>210.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>211.2</td>\n",
       "      <td>217.6</td>\n",
       "      <td>208.0</td>\n",
       "      <td>238.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>174.4</td>\n",
       "      <td>185.2</td>\n",
       "      <td>203.6</td>\n",
       "      <td>214.8</td>\n",
       "      <td>204.8</td>\n",
       "      <td>210.8</td>\n",
       "      <td>201.6</td>\n",
       "      <td>230.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>170.4</td>\n",
       "      <td>181.2</td>\n",
       "      <td>198.8</td>\n",
       "      <td>210.0</td>\n",
       "      <td>200.4</td>\n",
       "      <td>206.0</td>\n",
       "      <td>197.2</td>\n",
       "      <td>225.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>167.2</td>\n",
       "      <td>177.6</td>\n",
       "      <td>195.2</td>\n",
       "      <td>205.6</td>\n",
       "      <td>196.8</td>\n",
       "      <td>202.4</td>\n",
       "      <td>193.6</td>\n",
       "      <td>221.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>163.6</td>\n",
       "      <td>174.0</td>\n",
       "      <td>190.8</td>\n",
       "      <td>201.6</td>\n",
       "      <td>192.4</td>\n",
       "      <td>198.0</td>\n",
       "      <td>189.2</td>\n",
       "      <td>216.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>154.0</td>\n",
       "      <td>164.8</td>\n",
       "      <td>181.2</td>\n",
       "      <td>190.4</td>\n",
       "      <td>182.0</td>\n",
       "      <td>185.2</td>\n",
       "      <td>178.4</td>\n",
       "      <td>204.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>147.2</td>\n",
       "      <td>158.0</td>\n",
       "      <td>173.2</td>\n",
       "      <td>182.4</td>\n",
       "      <td>174.8</td>\n",
       "      <td>177.6</td>\n",
       "      <td>170.8</td>\n",
       "      <td>196.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>141.6</td>\n",
       "      <td>152.0</td>\n",
       "      <td>166.8</td>\n",
       "      <td>176.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>171.2</td>\n",
       "      <td>164.4</td>\n",
       "      <td>189.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>136.4</td>\n",
       "      <td>146.0</td>\n",
       "      <td>160.8</td>\n",
       "      <td>169.2</td>\n",
       "      <td>161.6</td>\n",
       "      <td>164.8</td>\n",
       "      <td>158.4</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>136.0</td>\n",
       "      <td>145.6</td>\n",
       "      <td>160.4</td>\n",
       "      <td>169.2</td>\n",
       "      <td>161.2</td>\n",
       "      <td>164.8</td>\n",
       "      <td>158.0</td>\n",
       "      <td>217.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>300 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0      1      2      3      4      5      6      7\n",
       "0    180.0  191.2  210.0  222.0  211.2  217.6  208.0  238.0\n",
       "1    174.4  185.2  203.6  214.8  204.8  210.8  201.6  230.4\n",
       "2    170.4  181.2  198.8  210.0  200.4  206.0  197.2  225.2\n",
       "3    167.2  177.6  195.2  205.6  196.8  202.4  193.6  221.2\n",
       "4    163.6  174.0  190.8  201.6  192.4  198.0  189.2  216.4\n",
       "..     ...    ...    ...    ...    ...    ...    ...    ...\n",
       "295  154.0  164.8  181.2  190.4  182.0  185.2  178.4  204.4\n",
       "296  147.2  158.0  173.2  182.4  174.8  177.6  170.8  196.0\n",
       "297  141.6  152.0  166.8  176.0  168.0  171.2  164.4  189.2\n",
       "298  136.4  146.0  160.8  169.2  161.6  164.8  158.4  182.0\n",
       "299  136.0  145.6  160.4  169.2  161.2  164.8  158.0  217.6\n",
       "\n",
       "[300 rows x 8 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(train24_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    0.353408\n",
       "3    0.350473\n",
       "0    0.345148\n",
       "4    0.341753\n",
       "2    0.340716\n",
       "7    0.340712\n",
       "5    0.339354\n",
       "6    0.337812\n",
       "dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 24 Training - Create Dataframe objects and compute correlations\n",
    "train24_data_df = pd.DataFrame(train24_data)\n",
    "train24_label_s = pd.Series(train24_label)\n",
    "train24_data_df.corrwith(train24_label_s).sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.386344\n",
       "6    0.375283\n",
       "2    0.374979\n",
       "3    0.368546\n",
       "1    0.364131\n",
       "5    0.362116\n",
       "7    0.346136\n",
       "4    0.339579\n",
       "dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 12 Training - Create Dataframe objects and compute correlations\n",
    "train12_data_df = pd.DataFrame(train12_data)\n",
    "train12_label_s = pd.Series(train12_label)\n",
    "train12_data_df.corrwith(train12_label_s).sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5    0.632901\n",
       "6    0.624794\n",
       "2    0.621514\n",
       "7    0.619775\n",
       "3    0.619472\n",
       "4    0.612446\n",
       "1    0.608329\n",
       "0    0.607035\n",
       "dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 6 Training - Create Dataframe objects and compute correlations\n",
    "train6_data_df = pd.DataFrame(train6_data)\n",
    "train6_label_s = pd.Series(train6_label)\n",
    "train6_data_df.corrwith(train6_label_s).sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Correlation between features seems to be about equal for predictions with training and test splits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuning KNeighbors Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training chips num:  [13  5 30 29 31  7 21 23  6 18  9  3 15  2 28 11 19 10  8 22 17 33 14  4]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [25  7  2 17 13 20  8  3  9 26 33 21]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 3 32  7 31  5 26]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 8}\n",
      "Best Accuracy:  0.9400000000000001\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  0.9400000000000001\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  1.0\n",
      "Training chips num:  [16  7  3 29 28 24 27 14 23 19 13  6 12 26 18  8 22  9 25 33 10  2  4 31]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 2 20 24 22 19  8  9 33  7 29  3 32]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 5  6 30 18 28 15]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 6}\n",
      "Best Accuracy:  0.9566666666666667\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 5}\n",
      "Best Accuracy:  0.8866666666666667\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9333333333333333\n",
      "Training chips num:  [26 29 28  5 16 18  2 30 19 11 27 15 32  6  9  4 17 14 23 33  8 24 20 21]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [30  2 20 21 18 25 12 27 16  4  5 10]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [15 24  3 10 16 26]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 8}\n",
      "Best Accuracy:  0.95\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9466666666666667\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 5}\n",
      "Best Accuracy:  1.0\n",
      "Training chips num:  [21 27 10  4 20 25 30 23  7 15  5  6 11 26 33 14  1 13  9 32 18  2 31 17]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [10  9 16 23 19  8 21 28 14 18 30 27]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [14 30 10 19 15 33]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 5}\n",
      "Best Accuracy:  0.9633333333333333\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9133333333333333\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  1.0\n",
      "Training chips num:  [33 28 29 24  8  5 19 32 15 18 25  6  9 17 30 22 11 31  2 23 10  4  1 16]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [10 12 16 29 20 30 13  1  4 11 27 32]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 6 30 24 26 33 22]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  0.9533333333333334\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9133333333333333\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  1.0\n",
      "Training chips num:  [27 26 13 15 31 29  6 12 22  1  9  2  8 32 17 23 24 19  7 11  3 33 14 16]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [28 11 27  9 31 24 33 22  3 16  1 26]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [30  4  1 21 22 11]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9133333333333333\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9866666666666667\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9866666666666667\n",
      "Training chips num:  [ 6 11 14 15 22 19  4  2 20 26  7 17 32 13  3  9 33  1 21  5 28 18 16 30]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [15 17 21 22 12 26  5 32  7 10 11 29]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [17 16 25 12  4 15]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  0.9566666666666667\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 7}\n",
      "Best Accuracy:  0.9266666666666667\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9466666666666667\n",
      "Training chips num:  [16 18  7 14 10 19 28 31  5  6  4 23 22 15 25 20 29 26 27 13 24 32  9  8]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [12 22  1  6 14 20 16 31 28 24 15 27]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [15 10  9 14 20 26]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9333333333333333\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 8}\n",
      "Best Accuracy:  0.9\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  1.0\n",
      "Training chips num:  [32 17 20 23 13  8 27  9  3  2 31  4 26 33  5 14 28 15  6 24 22 10 29  7]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 8 13 19  2 31 18 14  3  6  4 26 17]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [28 16 31  6 26  7]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TF', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 8}\n",
      "Best Accuracy:  0.9466666666666667\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9333333333333332\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9866666666666667\n",
      "Training chips num:  [14  8 31 24 28  2 26 25 13 18 23 22  6 15 16 20 12 17 11 30  4 33  1 32]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [29  7  5  8 30 23 24  1 18 31  2 13]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [25 22 14 13 27 11]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 5}\n",
      "Best Accuracy:  0.9466666666666667\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 5}\n",
      "Best Accuracy:  0.9533333333333334\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [33 21 32 10 20 13 30  3 19 16 24 31 29 23  1  4 14 11  6  9 27 18  5  7]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [22  3 11 31 15 28 10 23  6 33 18 20]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [26 24 17  7  2 23]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9233333333333332\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 9}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9066666666666666\n",
      "Training chips num:  [21 33  8 12 32 16  1 10  6  2 19  5 13  7 14 18 22 28 31 15  3 17 27 25]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [17 15 24  2 10 32  1 21 30 16 11 22]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [33  1 25  2 18 29]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  0.9133333333333333\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9066666666666668\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  1.0\n",
      "Training chips num:  [26 13  2  8 25 23 29 15 18 19 20 10 24 32 11 17 22 30  9 27  4  1 21  3]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [29 11 14 17 32 30 23  6 20  1 21 27]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 8  6 33  5 13 16]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 7}\n",
      "Best Accuracy:  0.9533333333333334\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 6}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "Training chips num:  [ 7  9 31 17 30 19  6 16 27  1  5  4 21  2  3  8 12 18 10 32 22 11 23 33]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [25 31  9 11 12 18 33 23 16 28 24 14]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [29 19 11  3  8 28]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9600000000000002\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9866666666666667\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [19 10 20 33  6  2 18 26 24 29 15 17  4 14 21 12 27  1  7 16 31 28  9 22]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 8 27 30 13 15 28 23  2  1 16  6 31]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [12  6 31 26 33 16]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9033333333333333\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 8}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [33 24 30  6  7  5 14 29 28 16 27 22  2  8 17 10 21  9 26 20  3 31 11 32]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [11 25 21 18 22  4 19 27 20  7 23 31]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [31  3  7  8 22  5]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 5}\n",
      "Best Accuracy:  0.9466666666666667\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  1.0\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "Training chips num:  [ 5 33 20 29 19 18  9 26 13 32 28 11  7 24  6 23  1 15  2 22  8 30 21 27]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [12 15  8 20 29 33 17  1 19 28  3 27]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [14 28  9  8 18  3]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9233333333333335\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9066666666666666\n",
      "Training chips num:  [11 24 22 10 14 17 25 29 13 20 27 21  4 15 26  1  7  6 18 30 28 16  5  2]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [31 10 25 19 27  8 20  4  7 33 24 26]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [17 28 29  3 18  4]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9033333333333333\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9066666666666666\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 5}\n",
      "Best Accuracy:  0.9333333333333332\n",
      "Training chips num:  [19 10 15  1 13 16 23 27 21 11 30 20 29 18  2  9 33 14  8  5 25 12  7 31]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [11 22 31 21  3  8  4 18 30 16 32  9]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [22 33 17 15 32 25]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.95\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 3}\n",
      "Best Accuracy:  0.9866666666666667\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [23 13 32 28  8 11 21 20  3 16 19 17 29  2 25 10 18  9  6 15 27 33 22 12]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [29 15 24 27  8 18 26  3 25 13 16  5]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [18  1 24  9 20  8]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 4}\n",
      "Best Accuracy:  0.9366666666666668\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 5}\n",
      "Best Accuracy:  0.9466666666666667\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 112 candidates, totalling 560 fits\n",
      "\n",
      " {'knc__leaf_size': 2, 'knc__n_neighbors': 2}\n",
      "Best Accuracy:  1.0\n"
     ]
    }
   ],
   "source": [
    "# Creating KNeighborsClassifier Pipeline\n",
    "KneighC = Pipeline([('scaler',StandardScaler()),('knc', KNeighborsClassifier())])\n",
    "\n",
    "# Finding the best parameters for KNeighborsClassifier\n",
    "\n",
    "## Create parameter grid for gridsearch algorithm\n",
    "knc_param_Grid=  { 'knc__n_neighbors': np.arange(2,10,1),\n",
    "                  'knc__leaf_size': np.arange(2,30,2),\n",
    "                 }\n",
    "\n",
    "## Create Grid search\n",
    "knc_grid_search = GridSearchCV(KneighC,\n",
    "                               param_grid=knc_param_Grid,\n",
    "                               scoring='accuracy',\n",
    "                               refit=True,\n",
    "                               cv=5,\n",
    "                               verbose=1)\n",
    "\n",
    "ii=0\n",
    "### Fit to training data\n",
    "KNC_trainscores=[]\n",
    "KNC_testscores=[]\n",
    "\n",
    "tpr_train_scores = []\n",
    "fpr_train_scores = []\n",
    "tpr_scores = []\n",
    "fpr_scores = []\n",
    "\n",
    "while ii < 20:\n",
    "\n",
    "    # Splits for each dataset\n",
    "    train24_data, train24_label, test24_data, test24_label = train_test_split(24,1,csv_path)\n",
    "    train12_data, train12_label, test12_data, test12_label = train_test_split(12,1,csv_path)\n",
    "    train6_data, train6_label, test6_data, test6_label = train_test_split(6,1,csv_path)\n",
    "    \n",
    "    train24_data = pd.DataFrame(train24_data)\n",
    "    test24_data = pd.DataFrame(test24_data)\n",
    "    \n",
    "    train12_data = pd.DataFrame(train12_data)\n",
    "    test12_data = pd.DataFrame(test12_data)\n",
    "    \n",
    "    train6_data = pd.DataFrame(train6_data)\n",
    "    test6_data = pd.DataFrame(test6_data)\n",
    "    \n",
    "\n",
    "    # Create lists of training sets to parse\n",
    "    training_sets = [train24_data, train12_data, train6_data]\n",
    "    training_labels = [train24_label, train12_label, train6_label]\n",
    "    \n",
    "    # Create lists of test sets to parse\n",
    "    test_sets = [test24_data, test12_data, test6_data]\n",
    "    test_labels =[test24_label, test12_label, test6_label]\n",
    "    \n",
    "    Sets = [\"-----24 Sample Training Set-----\",\"-----12 Sample Training Set-----\", \"-----6 Sample Training Set-----\"]\n",
    "    t_sets = [\"-----24 Sample Test Set-----\",\"-----12 Sample Test Set-----\", \"-----6 Sample Test Set-----\"]\n",
    "    \n",
    "    #Iterater\n",
    "    ii = ii + 1\n",
    "    \n",
    "    # Loop responsible for parsing training sets and training labels\n",
    "    for (a,i,j,m,n) in zip(Sets, training_sets,training_labels, test_sets, test_labels):\n",
    "        print(\"\\n\",a,\"\\n\")\n",
    "        knc_grid_search.fit(i,j)\n",
    "        ### Print Best parameters\n",
    "        print(\"\\n\",knc_grid_search.best_params_)\n",
    "        ## Saving the tuned model\n",
    "        model = knc_grid_search.best_estimator_\n",
    "        print(\"Best Accuracy: \",knc_grid_search.best_score_)\n",
    "        KNC_trainscores+= [knc_grid_search.best_score_]\n",
    "        y_pred = model.predict(m)\n",
    "        KNC_testscores += [accuracy_score(n,y_pred)]\n",
    "        \n",
    "        # Calculate Average TPR and Average FPR for training set\n",
    "        tn, fp, fn, tp = confusion_matrix(j, model.predict(i)).ravel()\n",
    "        tpr_train = tp/(tp+fn)\n",
    "        fpr_train = fp/(fp+tn)\n",
    "\n",
    "        # Create list of TPR and FPR for train\n",
    "        tpr_train_scores += [tpr_train]\n",
    "        fpr_train_scores += [fpr_train]\n",
    "\n",
    "        # Calculate Average TPR and Average FPR for test set\n",
    "        tn, fp, fn, tp = confusion_matrix(n, y_pred).ravel()\n",
    "        tpr_test = tp/(tp+fn)\n",
    "        fpr_test = fp/(fp+tn)\n",
    "        \n",
    "        # Create list of TPR and FPR\n",
    "        tpr_scores += [tpr_test]\n",
    "        fpr_scores += [fpr_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results for KneighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------- Kneighbors Accuracy Scores----------\n",
      "TRAIN24 Average Accuracy:  0.937\n",
      "Test24 Average Accruacy:  0.9331111111111111\n",
      "Train12 Average Accuracy:  0.9373333333333334\n",
      "Test12 Average Accruacy:  0.917047619047619\n",
      "TRAIN6 Average Accuracy:  0.9666666666666667\n",
      "Test6 Average Accruacy:  0.9082222222222223\n",
      "\n",
      " ---------- KNC Average TPR----------\n",
      "TRAIN24 Average TPR:  0.7958333333333333\n",
      "Test24 Average TPR:  0.48333333333333334\n",
      "Train12 Average TPR:  0.7708333333333334\n",
      "Test12 Average TPR:  0.5535714285714286\n",
      "TRAIN6 Average TPR:  0.9333333333333333\n",
      "Test6 Average TPR:  0.7277777777777777\n",
      "\n",
      " ---------- KNC Average FPR----------\n",
      "TRAIN24 Average FPR:  0.01213768115942029\n",
      "Test24 Average FPR:  0.027777777777777776\n",
      "Train12 Average FPR:  0.009782608695652175\n",
      "Test12 Average FPR:  0.05134575569358178\n",
      "TRAIN6 Average FPR:  0.005072463768115942\n",
      "Test6 Average FPR:  0.07608695652173914\n"
     ]
    }
   ],
   "source": [
    "# ----- Accuracy scores for test and training sets -----\n",
    "# Slicing training scores into individual lists\n",
    "KNC_24_train_scores = KNC_trainscores[0::3]\n",
    "KNC_12_train_scores= KNC_trainscores[1::3]\n",
    "KNC_6_train_scores = KNC_trainscores[2::3]\n",
    "\n",
    "# Slicing test scores into individual lists\n",
    "KNC_24_test_scores = KNC_testscores[0::3]\n",
    "KNC_12_test_scores = KNC_testscores[1::3]\n",
    "KNC_6_test_scores = KNC_testscores[2::3]\n",
    "\n",
    "print(\"---------- Kneighbors Accuracy Scores----------\")\n",
    "\n",
    "# Accuracy scores for test and training sets\n",
    "train24_avg = mean(KNC_24_train_scores)\n",
    "test24_avg = mean(KNC_24_test_scores)\n",
    "print(\"TRAIN24 Average Accuracy: \", train24_avg)\n",
    "print(\"Test24 Average Accruacy: \" , test24_avg)\n",
    "\n",
    "# Accuracy scores for test and training sets\n",
    "train12_avg = mean(KNC_12_train_scores)\n",
    "test12_avg = mean(KNC_12_test_scores)\n",
    "print(\"Train12 Average Accuracy: \", train12_avg)\n",
    "print(\"Test12 Average Accruacy: \" , test12_avg)\n",
    "\n",
    "# Accuracy Scores for test and training sets\n",
    "train6_avg = mean(KNC_6_train_scores)\n",
    "test6_avg = mean(KNC_6_test_scores)\n",
    "print(\"TRAIN6 Average Accuracy: \", train6_avg)\n",
    "print(\"Test6 Average Accruacy: \" , test6_avg)\n",
    "\n",
    "# ----- Average TPR and Average FPR -----\n",
    "# TRAINING SET\n",
    "KNC_24train_TPR = tpr_train_scores[0::3]\n",
    "KNC_12train_TPR = tpr_train_scores[1::3]\n",
    "KNC_6train_TPR = tpr_train_scores[2::3]\n",
    "KNC_24train_FPR = fpr_train_scores[0::3]\n",
    "KNC_12train_FPR = fpr_train_scores[1::3]\n",
    "KNC_6train_FPR = fpr_train_scores[2::3]\n",
    "\n",
    "\n",
    "# TEST SET\n",
    "KNC_24test_TPR = tpr_scores[0::3]\n",
    "KNC_12test_TPR = tpr_scores[1::3]\n",
    "KNC_6test_TPR = tpr_scores[2::3]\n",
    "KNC_24test_FPR = fpr_scores[0::3]\n",
    "KNC_12test_FPR = fpr_scores[1::3]\n",
    "KNC_6test_FPR = fpr_scores[2::3]\n",
    "\n",
    "print(\"\\n\",\"---------- KNC Average TPR----------\")\n",
    "TPR_train24 = mean(KNC_24train_TPR)\n",
    "TPR_test24 = mean(KNC_24test_TPR)\n",
    "print(\"TRAIN24 Average TPR: \", TPR_train24)\n",
    "print(\"Test24 Average TPR: \" , TPR_test24)\n",
    "TPR_train12 = mean(KNC_12train_TPR)\n",
    "TPR_test12 = mean(KNC_12test_TPR)\n",
    "print(\"Train12 Average TPR: \", TPR_train12)\n",
    "print(\"Test12 Average TPR: \" , TPR_test12)\n",
    "TPR_train6 = mean(KNC_6train_TPR)\n",
    "TPR_test6 = mean(KNC_6test_TPR)\n",
    "print(\"TRAIN6 Average TPR: \", TPR_train6)\n",
    "print(\"Test6 Average TPR: \" , TPR_test6)\n",
    "\n",
    "print(\"\\n\",\"---------- KNC Average FPR----------\")\n",
    "FPR_train24 = mean(KNC_24train_FPR)\n",
    "FPR_test24 = mean(KNC_24test_FPR)\n",
    "print(\"TRAIN24 Average FPR: \", FPR_train24)\n",
    "print(\"Test24 Average FPR: \" , FPR_test24)\n",
    "FPR_train12 = mean(KNC_12train_FPR)\n",
    "FPR_test12 = mean(KNC_12test_FPR)\n",
    "print(\"Train12 Average FPR: \", FPR_train12)\n",
    "print(\"Test12 Average FPR: \" , FPR_test12)\n",
    "FPR_train6 = mean(KNC_6train_FPR)\n",
    "FPR_test6 = mean(KNC_6test_FPR)\n",
    "print(\"TRAIN6 Average FPR: \", FPR_train6)\n",
    "print(\"Test6 Average FPR: \" , FPR_test6)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuning Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training chips num:  [20 11 13  5 12 28 19 27 21 26 31  3  4  6 29 24 22 23 14 30 33  2 15 16]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [32  3 28 21 10  9 19 23 29  1  5 27]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 3  2 26 29 11  1]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.9, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9933333333333334\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.8, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9866666666666667\n",
      "Training chips num:  [ 4 18 20  5 12 17  8  1 13  3 15 11 30 28  7 27  6 23  9 10 33 31 21 32]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 6 29 16 24 15  9 18 12 11 13 28 14]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [31  9  1  2  4 13]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [ 5 29 25 15  7 13 31 14 18  6 26 19  2 12  8 17 10 20  4 21 28 32 16 24]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 1 13 32  8 17 11 23 31  5  6  9  7]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [14 33  6  1 13 24]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 1.0, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [29  7  2 15  6 10 25  5 14 26 31 24 11 28 22 30  8 17 16  4 32 23  9 21]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [24 17  2  1 10 30 13 16 33 11  5  9]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [30 32 14 17  4 20]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.01, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9266666666666665\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.6, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9866666666666667\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9466666666666667\n",
      "Training chips num:  [26  4 16 23 30  5  2 25 29 10 19 22 24 12  1 14  7  3 15 11 33 31 28 13]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [31 25 26 20  6  9 22 28 15 10 19 17]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [27 18  3 14 30 28]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.01, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9266666666666665\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "Training chips num:  [23 19  4 20  1 10 24 32  3 13 27 22 14 21 25 33  9 15 30  2 28 26  7 29]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 3  1 10 13 26 17 12 32 21 18 20  5]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [19 21 25 24 27 33]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.01, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9266666666666665\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.7, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  1.0\n",
      "Training chips num:  [31 16 12 10 15  4 29 20 17 25 28 22 30 32 11  7 19 24  5 26 13 21  2 14]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [17 20  5  3 16 24  2 30  4  7 28 15]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [31 20 33 32 22  9]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.01, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9266666666666667\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.4, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [17 24 27 33 23 25 18 19 32 16 28 22 10 31  5  9 14 30 13  1 26 11 21  7]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 2 27 17 21 12  5  4 28 26  1 23  9]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [11 23 28  7 24  2]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.3, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9366666666666668\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9400000000000001\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.3, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  1.0\n",
      "Training chips num:  [20 30 27 10 19 12 28 26 11 18  3 31 16 25 24 22  7  1 17  8 13 15 21  9]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 4 30 22 18 26 23 13 33  5  7  6  8]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [21 17  9 12 26 22]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.5, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9366666666666665\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9333333333333333\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.2, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [27 31  5 17 15 14  9  1 32  3  6 18 26  7 22 10 19 13 30 20 28 24 12 23]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 2 14  5 11  7 21 13 30 24 12 23 17]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [16 28 29  5 14 20]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.01, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9266666666666667\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "Training chips num:  [13 18 16 24  9 23 11 25 26 12  5 20 19  8 17  2 14  6 22  3 21 33  1 15]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [29 23 11  1 20 27 21 32  7 13 28  8]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 2 17 30 10 29  1]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.2, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9466666666666667\n",
      "Training chips num:  [ 2 11 10 21  9 25  1  7  4 12 30 32 26 20 13  5 19 33 16 14 29 24 15 27]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 2 10  8 19  6  4 21 11 30 27 14 22]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [27 11  3 16  6 33]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "Training chips num:  [33 15  2 18 16  3 17 31 21 14 23 10 13 32 11  9 19  5 12 25 26 30  7 28]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [23 28 32 12  9 33  1 14 17 27 18 22]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 4 19  5 17 24  9]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 1.0, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9766666666666666\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.8, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9466666666666667\n",
      "Training chips num:  [10 30  1  3 31 21 13 18 29 15 19 11 20  5 24 23  4 33  9 26 17  7 28 27]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 6  7 17 30  1 11 19  5 13  4 26 10]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [16 28  9  3 15 10]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.01, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9266666666666665\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "Training chips num:  [33 11 30 27 25 26 23  8  3 16  2 13  1 14  6 32 10 29 17 19 15  7 20  9]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [ 2 32 10 33 25 15  6  9 31 14 20 26]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [22  9 15 32 13 26]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.01, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9233333333333332\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.3, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  1.0\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [12 20  6 27  9 26 31 19 23  7 30 18 21 25 24 28 15  8  2 29 17 32 14 33]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [21  3 31 12 18 10 27 22  9 15 32  1]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [30 24 20 10  9  1]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.01, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9266666666666667\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.3, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9866666666666667\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.2, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [ 1 20 25 15  9  8 29 26 12 21 17 13 33 24 11  3  2 18 16 23 10  5  4 27]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [15  1 30 10 22  7 29  6 27 14  2  5]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [29 13 25 20  7  6]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9333333333333332\n",
      "Training chips num:  [21 16  9  5 31 25  6 10 30 18  4 29 15  1  2 17 13 20 14  3  8  7 11 12]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [10 11 14 31 18  5  6 16 21 23 13 25]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 3 26  8 22 13  1]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.01, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9266666666666667\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9733333333333334\n",
      "Training chips num:  [13 30 24 29  1 12 18  4 17 27 21  6 10 31  8 15 16  7 32 23  2 25 20 11]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [21 13 27  2 25 14 26  1  7 20  3 24]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [27 16 20 26 19  8]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9200000000000002\n",
      "Training chips num:  [ 1 13 33 29  9  7  5 18 27 28 32  2 16 25 15  8 24 26 21  3  4 23 19 17]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [30  9  7 11 29 12  1 14 33  5 17  3]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [15  5 22 26 18  8]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.1, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9333333333333333\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Fitting 5 folds for each of 39 candidates, totalling 195 fits\n",
      "\n",
      " {'logreg__C': 0.001, 'logreg__tol': 0.0001}\n",
      "Best Accuracy:  0.9199999999999999\n"
     ]
    }
   ],
   "source": [
    "# Create Logisitic Regression Model\n",
    "Logreg = Pipeline([('scaler',StandardScaler()),('logreg',LogisticRegression(random_state=42))])\n",
    "Logreg\n",
    "\n",
    "# Finding the best parameters for Logistic Regression\n",
    "\n",
    "## Create Parameter Grid\n",
    "logreg_param_grid = {'logreg__C': [0.001,0.01,0.1,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0],\n",
    "                     'logreg__tol': [0.0001, 0.001, 0.01]\n",
    "                    }\n",
    "\n",
    "## Create Log Reg Gridsearch\n",
    "logreg_gridsearch = GridSearchCV(Logreg,\n",
    "                                 param_grid=logreg_param_grid,\n",
    "                                 scoring='accuracy',\n",
    "                                 refit=True,\n",
    "                                 cv=5,\n",
    "                                 verbose=1)\n",
    "\n",
    "### Fit to training data\n",
    "logreg_trainscores=[]\n",
    "logreg_testscores=[]\n",
    "\n",
    "# tpr and fpr scores\n",
    "tpr_scores = []\n",
    "fpr_scores = []\n",
    "\n",
    "tpr_train_scores = []\n",
    "fpr_train_scores = []\n",
    "tpr_test_scores = []\n",
    "fpr_test_scores = []\n",
    "\n",
    "ii=0\n",
    "while ii < 20:\n",
    "\n",
    "    # Splits for each dataset\n",
    "    train24_data, train24_label, test24_data, test24_label = train_test_split(24,1,csv_path)\n",
    "    train12_data, train12_label, test12_data, test12_label = train_test_split(12,1,csv_path)\n",
    "    train6_data, train6_label, test6_data, test6_label = train_test_split(6,1,csv_path)\n",
    "    \n",
    "    # Create pandas dataframes and series\n",
    "    train24_data = pd.DataFrame(train24_data)\n",
    "    test24_data = pd.DataFrame(test24_data)\n",
    "    \n",
    "    train12_data = pd.DataFrame(train12_data)\n",
    "    test12_data = pd.DataFrame(test12_data)\n",
    "    \n",
    "    train6_data = pd.DataFrame(train6_data)\n",
    "    test6_data = pd.DataFrame(test6_data)\n",
    "    \n",
    "    # Create lists of training sets to parse\n",
    "    training_sets = [train24_data, train12_data, train6_data]\n",
    "    training_labels = [train24_label, train12_label, train6_label]\n",
    "    \n",
    "    # Create lists of test sets to parse\n",
    "    test_sets = [test24_data, test12_data, test6_data]\n",
    "    test_labels =[test24_label, test12_label, test6_label]\n",
    "    \n",
    "    Sets = [\"-----24 Sample Training Set-----\",\"-----12 Sample Training Set-----\", \"-----6 Sample Training Set-----\"]\n",
    "    t_sets = [\"-----24 Sample Test Set-----\",\"-----12 Sample Test Set-----\", \"-----6 Sample Test Set-----\"]\n",
    "    \n",
    "    #Iterater\n",
    "    ii = ii + 1\n",
    "    \n",
    "  \n",
    "\n",
    "    # Loop responsible for parsing training sets and training labels\n",
    "    for (a,i,j,m,n) in zip(Sets, training_sets,training_labels, test_sets, test_labels):\n",
    "        print(\"\\n\",a,\"\\n\")\n",
    "        logreg_gridsearch.fit(i,j)\n",
    "        ### Print Best parameters\n",
    "        print(\"\\n\",logreg_gridsearch.best_params_)\n",
    "        ## Saving the tuned model\n",
    "        model = logreg_gridsearch.best_estimator_\n",
    "        print(\"Best Accuracy: \",logreg_gridsearch.best_score_)\n",
    "       \n",
    "        logreg_trainscores+= [logreg_gridsearch.best_score_]\n",
    "       \n",
    "        y_pred = model.predict(m)\n",
    "        logreg_testscores += [accuracy_score(n,y_pred)]\n",
    "\n",
    "        # Calculate TPR and FPR on training set\n",
    "        tp, fn, fp, tn = confusion_matrix(j,model.predict(i)).ravel()\n",
    "        fpr = fp/(fp+tn)\n",
    "        tpr = tp/(tp+fn)\n",
    "        \n",
    "        tpr_train_scores += [tpr]\n",
    "        fpr_train_scores += [fpr]\n",
    "        \n",
    "        # Calculate FPR and TPR on test set\n",
    "        tp, fn, fp, tn = confusion_matrix(n,y_pred).ravel()\n",
    "        fpr_test = fp/(fp+tn)\n",
    "        tpr_test = tp/(tp+fn)\n",
    "\n",
    "        tpr_test_scores += [tpr_test]\n",
    "        fpr_test_scores += [fpr_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log Reg Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------- Logistic Regression Accuracy Scores----------\n",
      "TRAIN24 Average Accuracy:  0.9316666666666666\n",
      "Test24 Average Accruacy:  0.9128888888888889\n",
      "Train12 Average Accuracy:  0.935\n",
      "Test12 Average Accruacy:  0.9159047619047619\n",
      "TRAIN6 Average Accuracy:  0.9546666666666667\n",
      "Test6 Average Accruacy:  0.9139259259259259\n",
      "---------- Logistic Regression Mean FPR and TPR Scores----------\n",
      "Mean FPR for 24 Sample Training Set:  0.80625\n",
      "Mean TPR for 24 Sample Training Set:  0.9963768115942029\n",
      "Mean FPR for 24 Sample Test Set:  0.8388888888888889\n",
      "Mean TPR for 24 Sample Test Set:  0.9782608695652174\n",
      "Mean FPR for 12 Sample Training Set:  0.7916666666666666\n",
      "Mean TPR for 12 Sample Training Set:  0.9992753623188406\n",
      "Mean FPR for 12 Sample Test Set:  0.8476190476190476\n",
      "Mean TPR for 12 Sample Test Set:  0.9822981366459628\n",
      "Mean FPR for 6 Sample Training Set:  0.5333333333333333\n",
      "Mean TPR for 6 Sample Training Set:  0.9992753623188406\n",
      "Mean FPR for 6 Sample Test Set:  0.6888888888888889\n",
      "Mean TPR for 6 Sample Test Set:  0.9663446054750402\n"
     ]
    }
   ],
   "source": [
    "# Slicing training scores into individual lists\n",
    "logreg_24_train_scores = logreg_trainscores[0::3]\n",
    "logreg_12_train_scores= logreg_trainscores[1::3]\n",
    "logreg_6_train_scores = logreg_trainscores[2::3]\n",
    "\n",
    "# Slicing test scores into individual lists\n",
    "logreg_24_test_scores = logreg_testscores[0::3]\n",
    "logreg_12_test_scores = logreg_testscores[1::3]\n",
    "logreg_6_test_scores = logreg_testscores[2::3]\n",
    "\n",
    "print(\"---------- Logistic Regression Accuracy Scores----------\")\n",
    "\n",
    "# Accuracy scores for test and training sets\n",
    "train24_avg = mean(logreg_24_train_scores)\n",
    "test24_avg = mean(logreg_24_test_scores)\n",
    "print(\"TRAIN24 Average Accuracy: \", train24_avg)\n",
    "print(\"Test24 Average Accruacy: \" , test24_avg)\n",
    "\n",
    "# Accuracy scores for test and training sets\n",
    "train12_avg = mean(logreg_12_train_scores)\n",
    "test12_avg = mean(logreg_12_test_scores)\n",
    "print(\"Train12 Average Accuracy: \", train12_avg)\n",
    "print(\"Test12 Average Accruacy: \" , test12_avg)\n",
    "\n",
    "# Accuracy Scores for test and training sets\n",
    "train6_avg = mean(logreg_6_train_scores)\n",
    "test6_avg = mean(logreg_6_test_scores)\n",
    "print(\"TRAIN6 Average Accuracy: \", train6_avg)\n",
    "print(\"Test6 Average Accruacy: \" , test6_avg)\n",
    "\n",
    "# Slicing fpr and tpr scores for each set\n",
    "logreg_24_fpr_train = fpr_train_scores[0::3]\n",
    "logreg_24_tpr_train = tpr_train_scores[0::3]\n",
    "logreg_12_fpr_train = fpr_train_scores[1::3]\n",
    "logreg_12_tpr_train = tpr_train_scores[1::3]\n",
    "logreg_6_fpr_train = fpr_train_scores[2::3]\n",
    "logreg_6_tpr_train = tpr_train_scores[2::3]\n",
    "\n",
    "logreg_24_fpr_test = fpr_test_scores[0::3]\n",
    "logreg_24_tpr_test = tpr_test_scores[0::3]\n",
    "logreg_12_fpr_test = fpr_test_scores[1::3]\n",
    "logreg_12_tpr_test = tpr_test_scores[1::3]\n",
    "logreg_6_fpr_test = fpr_test_scores[2::3]\n",
    "logreg_6_tpr_test = tpr_test_scores[2::3]\n",
    "\n",
    "\n",
    "# Printing Mean score for each set\n",
    "\n",
    "print(\"---------- Logistic Regression Mean FPR and TPR Scores----------\")\n",
    "\n",
    "# Mean FPR and TPR scores for training sets\n",
    "print(\"Mean FPR for 24 Sample Training Set: \", mean(logreg_24_fpr_train))\n",
    "print(\"Mean TPR for 24 Sample Training Set: \", mean(logreg_24_tpr_train))\n",
    "print(\"Mean FPR for 24 Sample Test Set: \", mean(logreg_24_fpr_test))\n",
    "print(\"Mean TPR for 24 Sample Test Set: \", mean(logreg_24_tpr_test))\n",
    "\n",
    "print(\"Mean FPR for 12 Sample Training Set: \", mean(logreg_12_fpr_train))\n",
    "print(\"Mean TPR for 12 Sample Training Set: \", mean(logreg_12_tpr_train))\n",
    "print(\"Mean FPR for 12 Sample Test Set: \", mean(logreg_12_fpr_test))\n",
    "print(\"Mean TPR for 12 Sample Test Set: \", mean(logreg_12_tpr_test))\n",
    "\n",
    "print(\"Mean FPR for 6 Sample Training Set: \", mean(logreg_6_fpr_train))\n",
    "print(\"Mean TPR for 6 Sample Training Set: \", mean(logreg_6_tpr_train))\n",
    "print(\"Mean FPR for 6 Sample Test Set: \", mean(logreg_6_fpr_test))\n",
    "print(\"Mean TPR for 6 Sample Test Set: \", mean(logreg_6_tpr_test))\n",
    "\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CASE 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to derive TPR and FPR\n",
    "def tpr_fpr(clusters,labels):\n",
    "    # Create confusion matrix\n",
    "    cm = confusion_matrix(clusters,labels)\n",
    "     # Derive TPR and FPR from confusion matrix\n",
    "    tpr = cm[1,1]/(cm[1,0]+cm[1,1])\n",
    "    fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n",
    "    return tpr,fpr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training chips num:  [27 25  4  1 16  3 24  2  9 12 19 13 11 20 17 15  8 31 23 21 18 28  7 29]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 11\n",
      "TF: 13\n",
      "Training chips num:  [30 14  4 31 29 24 27 19  1 25  3 15]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [25 31 20 10  9  7]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [21 32 33 11 28  3 16  4 29  8 12 20 10  6 19 22  1 18 15 31 30 24  5  7]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF']\n",
      "TI: 13\n",
      "TF: 11\n",
      "Training chips num:  [ 3 30 29  7 28 10 12 16  4 23 20 33]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [19 21 17 24  9 29]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [10 27 19  5  8 18 31 15 12  6 11 26 25 13 30  4  2 20 21  1 17  3 29 28]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [14 24  8  9 21 32 26  6 18 30 33  1]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI']\n",
      "TI: 9\n",
      "TF: 3\n",
      "Training chips num:  [31  1 19 23 16 11]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 2\n",
      "TF: 4\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [12 29 31 24  3 25 32 10 18  2  9 15  1 33 11  8  5  4 21 19 13 30 22  7]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 14\n",
      "TF: 10\n",
      "Training chips num:  [27 15 31  5  8 10 25 33 21  1  9 12]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [ 7 16 17 26 24 31]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [15 21  7 30 17  2  3 12  1  6 14  5 23 25 22 13 10  4 16 33 29 28 27  9]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 13\n",
      "TF: 11\n",
      "Training chips num:  [ 7 12 13  4  1 18 24  8 10  5 21 11]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 4\n",
      "TF: 8\n",
      "Training chips num:  [ 6 15 20 32  5 21]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [18  9 33 20 29  3  8 31  5 17  1 28  2 14 32 22 26 10 23 13 12  4 21 19]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 13\n",
      "TF: 11\n",
      "Training chips num:  [20 19 27  3 22  4 14 25 26 18  8 21]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 4\n",
      "TF: 8\n",
      "Training chips num:  [32  3 33 23 27 22]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [23 11 13  5  1 28  2 16 18  3 15  6 17 29 20 22  8  4 12 31 10 26 30 19]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 8\n",
      "TF: 16\n",
      "Training chips num:  [15 24  6  2 12 21 26 33 23 13 30  7]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 11\n",
      "TF: 1\n",
      "Training chips num:  [ 5  8 17  3 19 31]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [11 20  9 19 25  8 12  5 31 29  3 17 23 27 30 21 14 13 15  2 33 26 28 18]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 9\n",
      "TF: 15\n",
      "Training chips num:  [12 16 27 10 14  5 19 18 25 28  3 30]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [16 23 15 20 29 13]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [17 11 19 29 10 14 27 24 26  3  5 21 23 28  6 12 13 22  9 31 30 18 20  8]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 14\n",
      "TF: 10\n",
      "Training chips num:  [10  2 15 31 23 12  1 20 19 16 14 17]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [21 10 33 15  1 23]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [16 17 14 20  7 28 13  1 31  5 26 11 24 30 23 27 22 10  6 25 29 12  4  3]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 9\n",
      "TF: 15\n",
      "Training chips num:  [26 23 14 29 30 22 13 24 10 27  9 20]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [11 16 30 31 27 29]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [ 7  4 19 27 15  8 20 25 10 30 21  1  6  3 22 31 11 24 13  5 12  2 17 28]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI']\n",
      "TI: 20\n",
      "TF: 4\n",
      "Training chips num:  [13 33 11 10 27  2  1  8 21 16 30 22]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI']\n",
      "TI: 8\n",
      "TF: 4\n",
      "Training chips num:  [23  1 28 27  7 22]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [12 33  4 19 14 10 23 16  3 29 15 30 13  7  9 24  2  6 20  8 26  5 17 27]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 9\n",
      "TF: 15\n",
      "Training chips num:  [23  4  8 29 20  5 28 16 11  3 33  6]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [12 18 14 21 17 19]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [30 14 24 26 15  8 13 25 23 33 16  2 32  9  7 22  5 19 27 18 31 11  3  4]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 10\n",
      "TF: 14\n",
      "Training chips num:  [ 2 12 30 28 29 23 15 25 18  5 21 17]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [20 10 32 23 30 11]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [13  4 15 25 23 27 21 12 29  2  6  3 14  8  9 20 24  1 30 10  7 19 28 22]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 9\n",
      "TF: 15\n",
      "Training chips num:  [ 9 32 19 22 28 15 26  8 33 23 11 17]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [18 20 19  5 25 33]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 2\n",
      "TF: 4\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [ 2 29 21 22 19 31 13 17 18 20 27  5 23 14 26  6 11  1 28 24 10  4  3 32]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 13\n",
      "TF: 11\n",
      "Training chips num:  [24 25 30 32 23 29  6  1  9 26  5 31]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 3\n",
      "TF: 9\n",
      "Training chips num:  [ 2 26  8 30 13 14]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 2\n",
      "TF: 4\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [ 2  9 12 19 33  5 29  8 30 20 32 15  7  3 31 14  1 28 13  4 25 18 21 27]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 14\n",
      "TF: 10\n",
      "Training chips num:  [32 26  6 17 14  5 27  1 13 30 18  7]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [26 28  4 30  3 29]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [27 21 28 31 29 10  6 23  2 18 19  8 22 13 14 15 16 12 20  5  7  4 25 26]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 15\n",
      "TF: 9\n",
      "Training chips num:  [ 3 27 33 19  8  4 11 20  1 32 22 16]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [11 20  6 12 32 25]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [ 9 18  7 24 21 17 33 23 16 31 27  1  3 32 14 13 29 20 25 19 28 10  5  6]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI']\n",
      "TI: 14\n",
      "TF: 10\n",
      "Training chips num:  [21  8  5 14 23  4 19 11 15  9 22 30]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [ 4  8 26  7  2 16]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [12 14 27 32  9 25 22 11 18 26 19 23  8 17 21 20 16 15 30  3 31  6 28 33]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 14\n",
      "TF: 10\n",
      "Training chips num:  [27 13 33  6 19 16  5 12 20 17 15  8]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [13  4 11 26 25 24]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [19 28  1  8 27 26  9 30 15 20 32  4 13  6  3 22 25 23 24 11  7 31  5 14]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 18\n",
      "TF: 6\n",
      "Training chips num:  [32 13 12 19 30 33 21  4  7 23 18  1]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [ 8 13  4 20  3  1]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 5\n",
      "TF: 1\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create KMeans model\n",
    "\n",
    "# Kmeans pipeline with scaler and pca\n",
    "\n",
    "\n",
    "\n",
    "kmeans = Pipeline([('scaler',StandardScaler()),('kmeans',KMeans(random_state=42))])\n",
    "\n",
    "### kmeans accuracy score lists\n",
    "kmeans_trainscores=[]\n",
    "kmeans_testscores=[]\n",
    "\n",
    "# tpr and fpr scores\n",
    "tpr_scores = []\n",
    "fpr_scores = []\n",
    "\n",
    "tpr_train_scores = []\n",
    "fpr_train_scores = []\n",
    "tpr_test_scores = []\n",
    "fpr_test_scores = []\n",
    "\n",
    "ii=0\n",
    "while ii < 20:\n",
    "\n",
    "    # Splits for each dataset\n",
    "    train24_data, train24_label, test24_data, test24_label = train_test_split(24,3,csv_path)\n",
    "    train12_data, train12_label, test12_data, test12_label = train_test_split(12,3,csv_path)\n",
    "    train6_data, train6_label, test6_data, test6_label = train_test_split(6,3,csv_path)\n",
    "    \n",
    "    # Create pandas dataframes and series\n",
    "    train24_data = pd.DataFrame(train24_data)\n",
    "    test24_data = pd.DataFrame(test24_data)\n",
    "    \n",
    "    train12_data = pd.DataFrame(train12_data)\n",
    "    test12_data = pd.DataFrame(test12_data)\n",
    "    \n",
    "    train6_data = pd.DataFrame(train6_data)\n",
    "    test6_data = pd.DataFrame(test6_data)\n",
    "    \n",
    "    # Create lists of training sets to parse\n",
    "    training_sets = [train24_data, train12_data, train6_data]\n",
    "    training_labels = [train24_label, train12_label, train6_label]\n",
    "    \n",
    "    # Create lists of test sets to parse\n",
    "    test_sets = [test24_data, test12_data, test6_data]\n",
    "    test_labels =[test24_label, test12_label, test6_label]\n",
    "    \n",
    "    Sets = [\"-----24 Sample Training Set-----\",\"-----12 Sample Training Set-----\", \"-----6 Sample Training Set-----\"]\n",
    "    t_sets = [\"-----24 Sample Test Set-----\",\"-----12 Sample Test Set-----\", \"-----6 Sample Test Set-----\"]\n",
    "    \n",
    "    #Iterater\n",
    "    ii = ii + 1\n",
    "\n",
    "    # Loop responsible for parsing training sets and training labels\n",
    "    for (a,i,j,m,n) in zip(Sets, training_sets,training_labels, test_sets, test_labels):\n",
    "        print(\"\\n\",a,\"\\n\")\n",
    "        j = [int(i) for i in j]\n",
    "        # Fit the model\n",
    "        kmeans.fit(i)\n",
    "\n",
    "        # Get clusters from training and test set\n",
    "        train_clusters = kmeans.predict(i)\n",
    "        test_clusters = kmeans.predict(m)\n",
    "\n",
    "        # Compute accuracy score between training cluster labels and training labels\n",
    "        train_score = accuracy_score(train_clusters,j)\n",
    "        \n",
    "        # Compute accuracy score between test cluster labels and test labels\n",
    "        test_score = accuracy_score(test_clusters,n)\n",
    "\n",
    "        # Appeniding to kmeans_train_scores list\n",
    "        kmeans_trainscores.append(train_score)\n",
    "\n",
    "        # Appending to kmeans_test_scores list\n",
    "        kmeans_testscores.append(test_score)\n",
    "\n",
    "        # derive TPR and TPR from training clusters and training labels\n",
    "        tpr_train, fpr_train = tpr_fpr(train_clusters,j)\n",
    "        tpr_train_scores.append(tpr_train)\n",
    "        fpr_train_scores.append(fpr_train)\n",
    "\n",
    "        # derive TPR and TPR from test clusters and test labels\n",
    "        tpr_test, fpr_test = tpr_fpr(test_clusters,n)\n",
    "        tpr_test_scores.append(tpr_test)\n",
    "        fpr_test_scores.append(fpr_test)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------- KMEANS Accuracy Scores----------\n",
      "Train24 Average Accuracy:  0.15515610032870417\n",
      "Test24 Average Accruacy:  0.15355555555555556\n",
      "Train12 Average Accuracy:  0.15256653600013065\n",
      "Test12 Average Accruacy:  0.1417142857142857\n",
      "Train6 Average Accuracy:  0.1296628787878788\n",
      "Test6 Average Accruacy:  0.12814814814814815\n",
      "---------- KMEANS Mean FPR and TPR Scores----------\n",
      "Kmeans 24 Sample Training Set FPR:  0.08357890193482226\n",
      "Kmeans 24 Sample Training Set TPR:  0.067448501953052\n",
      "Kmeans 24 Sample Test Set FPR:  0.1285144927536232\n",
      "Kmeans 24 Sample Test Set TPR:  0.06514188940675666\n",
      "Kmeans 12 Sample Training Set FPR:  0.0791592394533571\n",
      "Kmeans 12 Sample Training Set TPR:  0.09239130434782608\n",
      "Kmeans 12 Sample Test Set FPR:  0.09028777566200641\n",
      "Kmeans 12 Sample Test Set TPR:  0.07460426002057094\n",
      "Kmeans 6 Sample Training Set FPR:  0.20984848484848484\n",
      "Kmeans 6 Sample Training Set TPR:  0.1297222222222222\n",
      "Kmeans 6 Sample Test Set FPR:  0.13761192340563672\n",
      "Kmeans 6 Sample Test Set TPR:  0.11935865809105246\n"
     ]
    }
   ],
   "source": [
    "# Slicing training scores into individual lists\n",
    "kmeans_24_train_scores = kmeans_trainscores[0::3]\n",
    "kmeans_12_train_scores = kmeans_trainscores[1::3]\n",
    "kmeans_6_train_scores = kmeans_trainscores[2::3]\n",
    "\n",
    "# Slicing test scores into individual lists\n",
    "kmeans_24_test_scores = kmeans_testscores[0::3]\n",
    "kmeans_12_test_scores = kmeans_testscores[1::3]\n",
    "kmeans_6_test_scores = kmeans_testscores[2::3]\n",
    "\n",
    "\n",
    "print(\"---------- KMEANS Accuracy Scores----------\")\n",
    "\n",
    "# Accuracy scores for test and training sets\n",
    "train24_avg = mean(kmeans_24_train_scores)\n",
    "test24_avg = mean(kmeans_24_test_scores)\n",
    "print(\"Train24 Average Accuracy: \", train24_avg)\n",
    "print(\"Test24 Average Accruacy: \" , test24_avg)\n",
    "\n",
    "# Accuracy scores for test and training sets\n",
    "train12_avg = mean(kmeans_12_train_scores)\n",
    "test12_avg = mean(kmeans_12_test_scores)\n",
    "print(\"Train12 Average Accuracy: \", train12_avg)\n",
    "print(\"Test12 Average Accruacy: \" , test12_avg)\n",
    "\n",
    "# Accuracy Scores for test and training sets\n",
    "train6_avg = mean(kmeans_6_train_scores)\n",
    "test6_avg = mean(kmeans_6_test_scores)\n",
    "print(\"Train6 Average Accuracy: \", train6_avg)\n",
    "print(\"Test6 Average Accruacy: \" , test6_avg)\n",
    "\n",
    "# Slicing fpr and tpr scores for each set\n",
    "kmeans_24_fpr = fpr_test_scores[0::3]\n",
    "kmeans_24_tpr = tpr_test_scores[0::3]\n",
    "kmeans_12_tpr = tpr_test_scores[1::3]\n",
    "kmeans_12_fpr = fpr_test_scores[1::3]\n",
    "kmeans_6_tpr = tpr_test_scores[2::3]\n",
    "kmeans_6_fpr = fpr_test_scores[2::3]\n",
    "\n",
    "\n",
    "# Printing Mean score for each set\n",
    "\n",
    "print(\"---------- KMEANS Mean FPR and TPR Scores----------\")\n",
    "\n",
    "# Mean FPR and TPR scores for kmeans training set\n",
    "kmeans_24_fpr_train = mean(fpr_train_scores[0::3])\n",
    "kmeans_24_tpr_train = mean(tpr_train_scores[0::3])\n",
    "kmeans_12_fpr_train = mean(fpr_train_scores[1::3])\n",
    "kmeans_12_tpr_train = mean(tpr_train_scores[1::3])\n",
    "kmeans_6_fpr_train = mean(fpr_train_scores[2::3])\n",
    "kmeans_6_tpr_train = mean(tpr_train_scores[2::3])\n",
    "\n",
    "# Mean FPR and TPR scores for kmeans test set\n",
    "# Find the average between each roc curve in fpr_test_scores[0::3]\n",
    "kmeans_24_fpr_test = mean(fpr_test_scores[0::3])\n",
    "kmeans_24_tpr_test = mean(tpr_test_scores[0::3])\n",
    "kmeans_12_fpr_test = mean(fpr_test_scores[1::3])\n",
    "kmeans_12_tpr_test = mean(tpr_test_scores[1::3])\n",
    "kmeans_6_fpr_test = mean(fpr_test_scores[2::3])\n",
    "kmeans_6_tpr_test = mean(tpr_test_scores[2::3])\n",
    "\n",
    "# Printing Mean FPR and TPR scores for each set\n",
    "print(\"Kmeans 24 Sample Training Set FPR: \", kmeans_24_fpr_train)\n",
    "print(\"Kmeans 24 Sample Training Set TPR: \", kmeans_24_tpr_train)\n",
    "print(\"Kmeans 24 Sample Test Set FPR: \", kmeans_24_fpr_test)\n",
    "print(\"Kmeans 24 Sample Test Set TPR: \", kmeans_24_tpr_test)\n",
    "print(\"Kmeans 12 Sample Training Set FPR: \", kmeans_12_fpr_train)\n",
    "print(\"Kmeans 12 Sample Training Set TPR: \", kmeans_12_tpr_train)\n",
    "print(\"Kmeans 12 Sample Test Set FPR: \", kmeans_12_fpr_test)\n",
    "print(\"Kmeans 12 Sample Test Set TPR: \", kmeans_12_tpr_test)\n",
    "print(\"Kmeans 6 Sample Training Set FPR: \", kmeans_6_fpr_train)\n",
    "print(\"Kmeans 6 Sample Training Set TPR: \", kmeans_6_tpr_train)\n",
    "print(\"Kmeans 6 Sample Test Set FPR: \", kmeans_6_fpr_test)\n",
    "print(\"Kmeans 6 Sample Test Set TPR: \", kmeans_6_tpr_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DBSCAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;scaler&#x27;, StandardScaler()), (&#x27;dbscan&#x27;, DBSCAN())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;scaler&#x27;, StandardScaler()), (&#x27;dbscan&#x27;, DBSCAN())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DBSCAN</label><div class=\"sk-toggleable__content\"><pre>DBSCAN()</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('scaler', StandardScaler()), ('dbscan', DBSCAN())])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create DBSCAN Model\n",
    "dbscan = Pipeline([('scaler',StandardScaler()),('dbscan', DBSCAN())])\n",
    "dbscan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training chips num:  [11 13 31  3  4 22 16 23  2  9 12 26 20 28 29 14 21  8 15 18 33 17 10 32]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 11\n",
      "TF: 13\n",
      "Training chips num:  [21 15 26  6 33 22 20 13 24  8  4 27]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 9\n",
      "TF: 3\n",
      "Training chips num:  [12 21 30 22 16  3]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TI', 'TF', 'TI']\n",
      "TI: 5\n",
      "TF: 1\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [22 14  1 32 30  4 31 29 21  8 24 19 17  6 20 26 28  5  2 10 18  7 12 15]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 8\n",
      "TF: 16\n",
      "Training chips num:  [11 13  9  1  6  8 15 17 14 31 21 20]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [15 11 18  4 30 24]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [22  7 29 30  6 16  5 19 12 31 11 17 32 21 20  2  8 25 18  1  9 14 28 27]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 14\n",
      "TF: 10\n",
      "Training chips num:  [32  2 19 31 16 17 10 18 30 23 33 12]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [20 24  8 14 31  1]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 2\n",
      "TF: 4\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [22 33 30 26  1  3 23 15 32 29 20  6 18 12  9 17 25  8 16 31  7  2 21 11]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [32 25 22  5 18 21 19 31 11  7 12  2]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [15  2 31 20 29 32]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 2\n",
      "TF: 4\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [ 4 25  5 31 24 29 14 27 19  2 10 33 20 30 13 23 18 21  3 15 26  9 12 17]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 13\n",
      "TF: 11\n",
      "Training chips num:  [ 3  2 16 22 18 26  9 11 27 12 31  4]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [28 22 27 13 23 20]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 2\n",
      "TF: 4\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [33  6 29 23 12  2 31 25 18 26 28 15 32 22  8 21  4 16 24  9 19 17  3 13]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 10\n",
      "TF: 14\n",
      "Training chips num:  [26  3 15 16 25  1 31  6 30 14  2 21]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 4\n",
      "TF: 8\n",
      "Training chips num:  [ 9 28 16 24  2 17]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [32 15 17  6  4 33 13  1 22 18 28  3 21 31  2  8 19 12  7 24 23  5 14 10]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [31 24  6 10 26  5 27 29 30  1  9 32]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [20  4 32  9 30 23]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [ 5  3 16 29 13 20 27 31  2 21  4 28 19 17 25 11  6 26 14 24 15 22 23 33]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 14\n",
      "TF: 10\n",
      "Training chips num:  [23 33 25 10  3 26 31  9 28 14 19  1]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [23  8 17 26 33  3]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [ 1  2  4 14 24 10 22  3 16  7 30  9 23 26 15 12 21 19 20 29 28  6 25 13]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [33  5 19 21 10 26  4 14 28 12 23  1]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [ 6 13 19  9 32 28]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 1\n",
      "TF: 5\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [24 15 11 27  8 28 30 20  7  9  4 21 23 19 29 12  6 16 22  2 17 25 10 18]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI']\n",
      "TI: 14\n",
      "TF: 10\n",
      "Training chips num:  [29  6 15 27 18 28 13 11 17 33 24  4]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [ 7 22 23 13 29 31]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 2\n",
      "TF: 4\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [ 5 12 21 18  7 29  1 33  3 19  9 23 26 11 30 25 22 24 15  8  6 17 32 28]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 10\n",
      "TF: 14\n",
      "Training chips num:  [29 33  7 18 30 27 23 12 20 16  8 26]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [26 28 17  5 10 25]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:6: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  tpr = cm[1,1]/(cm[1,0]+cm[1,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [26 17 33 10  6 19 14  3 27 18 31 25 20 15 29 28 24 30 21 23  4 22  1 13]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI']\n",
      "TI: 19\n",
      "TF: 5\n",
      "Training chips num:  [14 31 19 13  1 15 11 16 23 20 30 21]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [18 13 30 16 21  2]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TF', 'TF']\n",
      "TI: 2\n",
      "TF: 4\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [24 30 10 18  5 33  9 25 26 12 13  4  6 17 15 31  1 16 19 20 23  3  2 32]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [27  7 32  1 18  6  9 13 19  8  2 22]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI']\n",
      "TI: 6\n",
      "TF: 6\n",
      "Training chips num:  [ 7 25 19 26 24 27]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [33  8 24 10 30 13 14 32  1  3 12 11  4 16 21 31  2 18 15  9  6 26 27 28]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 8\n",
      "TF: 16\n",
      "Training chips num:  [28  3  1 32  9  5 30 17 12 18 20  8]\n",
      "Type selection: ['TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [30 29 20 24 28 12]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [ 9 10 24 20 22 33 23 31 17 32  1 11 26  7  5 12 14 27  4 16 28 21 13  3]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 8\n",
      "TF: 16\n",
      "Training chips num:  [11  2 28 10 20  4  5  9 13 24  3 22]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [19  8  2  9  3 14]\n",
      "Type selection: ['TF', 'TI', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 1\n",
      "TF: 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:6: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  tpr = cm[1,1]/(cm[1,0]+cm[1,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [33 24 12 21 28  6 22 25 19 27 20  5 14 30 17  8 15  3 31  7 13 23 11  2]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI']\n",
      "TI: 12\n",
      "TF: 12\n",
      "Training chips num:  [27 26 15 28 13 18  5 29 17 22  3 12]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 9\n",
      "TF: 3\n",
      "Training chips num:  [26 22 33 15  1 32]\n",
      "Type selection: ['TF', 'TF', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 3\n",
      "TF: 3\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [17  3 31 30  5  8 15 32 10  7 29 25 28 18 20 19 12 33 27  4 23  6 13  9]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI']\n",
      "TI: 13\n",
      "TF: 11\n",
      "Training chips num:  [17 26  4 10 30 14 15 12 32  6 23 29]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [19 10  4 15 23 32]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 0\n",
      "TF: 6\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [13  3 15 26 23 25 20  8 19 24 27  7 32 18 12 22 31 11 28 21  5 29 30  4]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF']\n",
      "TI: 11\n",
      "TF: 13\n",
      "Training chips num:  [29  7 26 32 12 28 31 20 18 33  8  9]\n",
      "Type selection: ['TI', 'TI', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF']\n",
      "TI: 7\n",
      "TF: 5\n",
      "Training chips num:  [ 5 14 20 18 30  2]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TI', 'TI']\n",
      "TI: 4\n",
      "TF: 2\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:6: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  tpr = cm[1,1]/(cm[1,0]+cm[1,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:6: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  tpr = cm[1,1]/(cm[1,0]+cm[1,1])\n",
      "C:\\Users\\patri\\AppData\\Local\\Temp\\ipykernel_16464\\217114975.py:7: RuntimeWarning: invalid value encountered in longlong_scalars\n",
      "  fpr = cm[0,1]/(cm[0,0]+cm[0,1])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training chips num:  [17 13 23 15  2 26 28 19  7  6  4 24  5 18 16 31 22 14 27  3 33 10 11  1]\n",
      "Type selection: ['TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF']\n",
      "TI: 11\n",
      "TF: 13\n",
      "Training chips num:  [25 11 29 24 15 14 32 17 16 33  6 27]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TI', 'TF', 'TF', 'TF', 'TF', 'TF', 'TF', 'TI']\n",
      "TI: 3\n",
      "TF: 9\n",
      "Training chips num:  [ 9  8 15 16 25 32]\n",
      "Type selection: ['TF', 'TF', 'TF', 'TI', 'TF', 'TF']\n",
      "TI: 1\n",
      "TF: 5\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n",
      "Training chips num:  [29 28 10 26 12 22 15 21  9 16  6  2 27  5  1 14 24 11  4 17 13 30 23 20]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TI', 'TF', 'TI', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TI', 'TF', 'TF', 'TF', 'TI', 'TI', 'TF', 'TF']\n",
      "TI: 11\n",
      "TF: 13\n",
      "Training chips num:  [24 12 11 25 16 18  4  2  6 13 14 26]\n",
      "Type selection: ['TI', 'TF', 'TF', 'TF', 'TF', 'TI', 'TI', 'TI', 'TF', 'TF', 'TI', 'TF']\n",
      "TI: 5\n",
      "TF: 7\n",
      "Training chips num:  [ 7  4 24 22 17 10]\n",
      "Type selection: ['TI', 'TF', 'TI', 'TI', 'TI', 'TI']\n",
      "TI: 5\n",
      "TF: 1\n",
      "\n",
      " -----24 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----12 Sample Training Set----- \n",
      "\n",
      "\n",
      " -----6 Sample Training Set----- \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create DBSCAN\n",
    "\n",
    "\n",
    "#DBSCAN pipeline\n",
    "dbscan = Pipeline([('scaler',StandardScaler()),('dbscan', DBSCAN())])\n",
    "\n",
    "\n",
    "### dbscan accuracy score lists\n",
    "dbscan_trainscores = []\n",
    "dbscan_testscores = []\n",
    "\n",
    "# tpr and fpr scores\n",
    "tpr_scores = []\n",
    "fpr_scores = []\n",
    "\n",
    "tpr_train_scores = []\n",
    "fpr_train_scores = []\n",
    "tpr_test_scores = []\n",
    "fpr_test_scores = []\n",
    "\n",
    "ii=0\n",
    "while ii < 20:\n",
    "\n",
    "\n",
    "\t# Splits for each dataset\n",
    "\ttrain24_data, train24_label, test24_data, test24_label = train_test_split(24,3,csv_path)\n",
    "\ttrain12_data, train12_label, test12_data, test12_label = train_test_split(12,3,csv_path)\n",
    "\ttrain6_data, train6_label, test6_data, test6_label = train_test_split(6,3,csv_path)\n",
    "\n",
    "\t# Create pandas dataframes and series\n",
    "\ttrain24_data = pd.DataFrame(train24_data)\n",
    "\ttest24_data = pd.DataFrame(test24_data)\n",
    "\ttrain12_data = pd.DataFrame(train12_data)\n",
    "\ttest12_data = pd.DataFrame(test12_data)\n",
    "\ttrain6_data = pd.DataFrame(train6_data)\n",
    "\ttest6_data = pd.DataFrame(test6_data)\n",
    "\n",
    "\t# Create lists of training sets to parse\n",
    "\ttraining_sets = [train24_data, train12_data, train6_data]\n",
    "\ttraining_labels = [train24_label, train12_label, train6_label]\n",
    "\n",
    "\t# Create lists of test sets to parse\n",
    "\ttest_sets = [test24_data, test12_data, test6_data]\n",
    "\ttest_labels =[test24_label, test12_label, test6_label]\n",
    "\tSets = [\"-----24 Sample Training Set-----\",\"-----12 Sample Training Set-----\", \"-----6 Sample Training Set-----\"]\n",
    "\tt_sets = [\"-----24 Sample Test Set-----\",\"-----12 Sample Test Set-----\", \"-----6 Sample Test Set-----\"]\n",
    "\n",
    "\t#Iterater\n",
    "\tii = ii + 1\n",
    "\n",
    "\t# Loop responsible for parsing training sets and training labels\n",
    "\tfor (a,i,j,m,n) in zip(Sets, training_sets,training_labels, test_sets, test_labels): \n",
    "\t\tprint(\"\\n\",a,\"\\n\")\n",
    "\t\tj = [int(i) for i in j]\n",
    "\t\t# Fit the model\n",
    "\t\tdbscan.fit(i)\n",
    "\n",
    "\t\t# Get clusters from training and test set\n",
    "\t\ttrain_clusters = dbscan.named_steps['dbscan'].labels_\n",
    "\n",
    "\t\t# Compute accuracy score between training cluster labels and training labels\n",
    "\t\ttrain_score = accuracy_score(train_clusters,j)\n",
    "\n",
    "\t\t# Appeniding to kmeans_train_scores list\n",
    "\n",
    "\t\t# if train score are nan, replace with 0\n",
    "\t\tdbscan_trainscores.append(train_score)\n",
    "\n",
    "\t\t# Fit model to test set\n",
    "\t\tdbscan.fit(m)\n",
    "\n",
    "\t\t# Compute test clusters\n",
    "\t\ttest_clusters = dbscan.named_steps['dbscan'].labels_\n",
    "\n",
    "\t\t# Compute accuracy score between test cluster labels and test labels\n",
    "\t\ttest_score = accuracy_score(test_clusters,n)\n",
    "\n",
    "\t\t# Appending to kmeans_test_scores list\n",
    "\t\t# if test score are nan, replace with 0\n",
    "\t\tdbscan_testscores.append(test_score)\n",
    "\n",
    "\t\t# derive TPR and TPR from training clusters and training labels\n",
    "\t\ttpr_train, fpr_train = tpr_fpr(train_clusters,j)\n",
    "\t\ttpr_train_scores.append(tpr_train)\n",
    "\t\tfpr_train_scores.append(fpr_train)\n",
    "\n",
    "\t\t# derive TPR and TPR from test clusters and test labels\n",
    "\t\ttpr_test, fpr_test = tpr_fpr(test_clusters,n)\n",
    "\t\ttpr_test_scores.append(tpr_test)\n",
    "\t\tfpr_test_scores.append(fpr_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------- DBSCAN Accuracy Scores----------\n",
      "Train24 Average Accuracy:  0.9059233295737945\n",
      "Test24 Average Accuracy:  0.8508888888888889\n",
      "Train12 Average Accuracy:  0.6965829517384017\n",
      "Test12 Average Accuracy:  0.9153333333333333\n",
      "Train6 Average Accuracy:  0.1296628787878788\n",
      "Test6 Average Accruacy:  0.12814814814814815\n",
      "---------- KMEANS Mean FPR and TPR Scores----------\n",
      "Train24 FPR:  1.0\n",
      "Train24 TPR:  1.0\n",
      "Test24 FPR:  0.9463163906339235\n",
      "Test24 TPR:  0.95\n",
      "Train12 FPR:  0.9417420814479638\n",
      "Train12 TPR:  0.9875\n",
      "Test12 FPR:  1.0\n",
      "Test12 TPR:  1.0\n",
      "Train6 FPR:  1.0\n",
      "Train6 TPR:  1.0\n",
      "Test6 FPR:  1.0\n",
      "Test6 TPR:  1.0\n"
     ]
    }
   ],
   "source": [
    "print(\"---------- DBSCAN Accuracy Scores----------\")\n",
    "\n",
    "# Slicing training scores into individual lists\n",
    "dbscan_24_train_scores = dbscan_trainscores[0::3]\n",
    "dbscan_12_train_scores = dbscan_trainscores[1::3]\n",
    "dbscan_6_train_scores = dbscan_trainscores[2::3]\n",
    "\n",
    "\n",
    "# Slicing test scores into individual lists\n",
    "dbscan_24_test_scores = dbscan_testscores[0::3]\n",
    "dbscan_12_test_scores = dbscan_testscores[1::3]\n",
    "dbscan_6_test_scores = dbscan_testscores[2::3]\n",
    "\n",
    "# Accuracy scores for test and training sets\n",
    "train24_avg = mean(dbscan_24_train_scores)\n",
    "test24_avg = mean(dbscan_24_test_scores)\n",
    "print(\"Train24 Average Accuracy: \", train24_avg)\n",
    "print(\"Test24 Average Accuracy: \", test24_avg)\n",
    "\n",
    "# Accuracy scores for test and training sets\n",
    "train12_avg = mean(dbscan_12_train_scores)\n",
    "test12_avg = mean(dbscan_12_test_scores)\n",
    "print(\"Train12 Average Accuracy: \", train12_avg)\n",
    "print(\"Test12 Average Accuracy: \", test12_avg)\n",
    "\n",
    "# Accuracy Scores for test and training sets\n",
    "train6_avg = mean(kmeans_6_train_scores)\n",
    "test6_avg = mean(kmeans_6_test_scores)\n",
    "print(\"Train6 Average Accuracy: \", train6_avg)\n",
    "print(\"Test6 Average Accruacy: \" , test6_avg)\n",
    "\n",
    "\n",
    "\n",
    "print(\"---------- KMEANS Mean FPR and TPR Scores----------\")\n",
    "\n",
    "# Slicing fpr and tpr scores for each set\n",
    "dbscan_24_fpr_test = fpr_test_scores[0::3]\n",
    "dbscan_24_tpr_test = tpr_test_scores[0::3]\n",
    "dbscan_12_tpr_test = tpr_test_scores[1::3]\n",
    "dbscan_12_fpr_test = fpr_test_scores[1::3]\n",
    "dbscan_6_tpr_test = tpr_test_scores[2::3]\n",
    "dbscan_6_fpr_test = fpr_test_scores[2::3]\n",
    "\n",
    "# training set slices\n",
    "dbscan_24_fpr_train = fpr_train_scores[0::3]\n",
    "dbscan_24_tpr_train = tpr_train_scores[0::3]\n",
    "dbscan_12_tpr_train = tpr_train_scores[1::3]\n",
    "dbscan_12_fpr_train = fpr_train_scores[1::3]\n",
    "dbscan_6_tpr_train = tpr_train_scores[2::3]\n",
    "dbscan_6_fpr_train = fpr_train_scores[2::3]\n",
    "\n",
    "# Drop the nans from training and test\n",
    "dbscan_24_fpr_train = [x for x in dbscan_24_fpr_train if str(x) != 'nan']\n",
    "dbscan_24_tpr_train = [x for x in dbscan_24_tpr_train if str(x) != 'nan']\n",
    "dbscan_12_tpr_train = [x for x in dbscan_12_tpr_train if str(x) != 'nan']\n",
    "dbscan_12_fpr_train = [x for x in dbscan_12_fpr_train if str(x) != 'nan']\n",
    "dbscan_6_tpr_train = [x for x in dbscan_6_tpr_train if str(x) != 'nan']\n",
    "dbscan_6_fpr_train = [x for x in dbscan_6_fpr_train if str(x) != 'nan']\n",
    "\n",
    "dbscan_24_fpr_test = [x for x in dbscan_24_fpr_test if str(x) != 'nan']\n",
    "dbscan_24_tpr_test = [x for x in dbscan_24_tpr_test if str(x) != 'nan']\n",
    "dbscan_12_tpr_test = [x for x in dbscan_12_tpr_test if str(x) != 'nan']\n",
    "dbscan_12_fpr_test = [x for x in dbscan_12_fpr_test if str(x) != 'nan']\n",
    "dbscan_6_tpr_test = [x for x in dbscan_6_tpr_test if str(x) != 'nan']\n",
    "dbscan_6_fpr_test = [x for x in dbscan_6_fpr_test if str(x) != 'nan']\n",
    "\n",
    "\n",
    "\n",
    "# Take the mean of the training and test set slices\n",
    "# training set slices\n",
    "dbscan_24_fpr_train_mean = mean(dbscan_24_fpr_train)\n",
    "dbscan_24_tpr_train_mean = mean(dbscan_24_tpr_train)\n",
    "dbscan_12_tpr_train_mean = mean(dbscan_12_tpr_train)\n",
    "dbscan_12_fpr_train_mean = mean(dbscan_12_fpr_train)\n",
    "dbscan_6_tpr_train_mean = mean(dbscan_6_tpr_train)\n",
    "dbscan_6_fpr_train_mean = mean(dbscan_6_fpr_train)\n",
    "\n",
    "dbscan_24_fpr_test_mean = mean(dbscan_24_fpr_test)\n",
    "dbscan_24_tpr_test_mean = mean(dbscan_24_tpr_test)\n",
    "dbscan_12_tpr_test_mean = mean(dbscan_12_tpr_test)\n",
    "dbscan_12_fpr_test_mean = mean(dbscan_12_fpr_test)\n",
    "dbscan_6_tpr_test_mean = mean(dbscan_6_tpr_test)\n",
    "dbscan_6_fpr_test_mean = mean(dbscan_6_fpr_test)\n",
    "\n",
    "# Printing training and test means\n",
    "print(\"Train24 FPR: \", dbscan_24_fpr_train_mean)\n",
    "print(\"Train24 TPR: \", dbscan_24_tpr_train_mean)\n",
    "print(\"Test24 FPR: \", dbscan_24_fpr_test_mean)\n",
    "print(\"Test24 TPR: \", dbscan_24_tpr_test_mean)\n",
    "\n",
    "print(\"Train12 FPR: \", dbscan_12_fpr_train_mean)\n",
    "print(\"Train12 TPR: \", dbscan_12_tpr_train_mean)\n",
    "print(\"Test12 FPR: \", dbscan_12_fpr_test_mean)\n",
    "print(\"Test12 TPR: \", dbscan_12_tpr_test_mean)\n",
    "\n",
    "print(\"Train6 FPR: \", dbscan_6_fpr_train_mean)\n",
    "print(\"Train6 TPR: \", dbscan_6_tpr_train_mean)\n",
    "print(\"Test6 FPR: \", dbscan_6_fpr_test_mean)\n",
    "print(\"Test6 TPR: \", dbscan_6_tpr_test_mean)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.15 ('eel4930')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  },
  "vscode": {
   "interpreter": {
    "hash": "9a793e4801943b75e92a711f3131632f9a1641d7c60df35a1a7f92c191e554b1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}