ensembling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from utils import *\n",
    "data_path = 'data/'\n",
    "oof_path =  data_path+'oof/'\n",
    "submission_path = data_path+ 'submission/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_X_y_ensembling(df,clip=20):\n",
    "    y = np.clip(df.target.values,0,clip)\n",
    "    X = df.drop(['target','date_block_num'],axis=1)\n",
    "    return X,y\n",
    "def get_oof_pickle(filename,clip=20):\n",
    "    return np.clip(pd.read_pickle(oof_path+filename),0,clip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import ElasticNetCV\n",
    "from sklearn.linear_model import ElasticNet,Ridge,LinearRegression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.ensemble import RandomForestRegressor\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate ensembling training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# lgb_oof = get_oof_pickle('lgb_best.pickle')\n",
    "\n",
    "# xgb2_oof = get_oof_pickle('xgb2.pickle')\n",
    "# xgb3_oof = get_oof_pickle('xgb3.pickle')\n",
    "\n",
    "# # fast_oof = get_oof_pickle('fastai1000500.pickle')\n",
    "\n",
    "# all_data = get_all_data(data_path,'new_sales_lag_after12.pickle')\n",
    "\n",
    "# all_data.target = np.clip(all_data.target.values,0,20)\n",
    "\n",
    "# temp = [lgb_oof,xgb2_oof,xgb3_oof,\n",
    "# #         fast_oof,\n",
    "#         lgb_oof-xgb2_oof,lgb_oof-xgb3_oof,\n",
    "# #         lgb_oof-fast_oof,\n",
    "#         xgb2_oof-xgb3_oof,\n",
    "# #         xgb2_oof-fast_oof,xgb3_oof-fast_oof,\n",
    "# #         np.mean(np.stack([lgb_oof,xgb2_oof,xgb3_oof,fast_oof],axis=1),axis=1)\n",
    "#         np.mean(np.stack([lgb_oof,xgb2_oof,xgb3_oof],axis=1),axis=1)]\n",
    "\n",
    "# all28 = all_data[all_data.date_block_num>=28].copy()\n",
    "\n",
    "# all_oof = np.stack(temp+[all28.target_lag_1,all28.date_block_num,all28.target],axis=1)\n",
    "\n",
    "# all_oof_df = pd.DataFrame(all_oof,columns=['lgb','xgb2','xgb3',\n",
    "# #                                            'fast',\n",
    "#                                            'lgb_xgb2','lgb_xgb3',\n",
    "# #                                            'lgb_fast',\n",
    "#                                            'xgb2_xgb3',\n",
    "# #                                            'xgb2_fast','xgb3_fast',\n",
    "#                                            'oof_mean','target_lag_1','date_block_num','target'])\n",
    "\n",
    "# all_oof_df.to_pickle(oof_path+'ensembling_nofast.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>lgb</th>\n",
       "      <td>0.378975</td>\n",
       "      <td>0.077147</td>\n",
       "      <td>0.107809</td>\n",
       "      <td>0.580430</td>\n",
       "      <td>1.194422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb2</th>\n",
       "      <td>0.451092</td>\n",
       "      <td>0.100030</td>\n",
       "      <td>0.142697</td>\n",
       "      <td>0.662918</td>\n",
       "      <td>1.272898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb3</th>\n",
       "      <td>0.477408</td>\n",
       "      <td>0.096444</td>\n",
       "      <td>0.137257</td>\n",
       "      <td>0.686700</td>\n",
       "      <td>1.174996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lgb_xgb2</th>\n",
       "      <td>-0.072118</td>\n",
       "      <td>-0.022882</td>\n",
       "      <td>-0.034888</td>\n",
       "      <td>-0.082488</td>\n",
       "      <td>-0.078476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lgb_xgb3</th>\n",
       "      <td>-0.098434</td>\n",
       "      <td>-0.019297</td>\n",
       "      <td>-0.029448</td>\n",
       "      <td>-0.106270</td>\n",
       "      <td>0.019426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb2_xgb3</th>\n",
       "      <td>-0.026316</td>\n",
       "      <td>0.003586</td>\n",
       "      <td>0.005440</td>\n",
       "      <td>-0.023782</td>\n",
       "      <td>0.097902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oof_mean</th>\n",
       "      <td>0.435825</td>\n",
       "      <td>0.091207</td>\n",
       "      <td>0.129255</td>\n",
       "      <td>0.643350</td>\n",
       "      <td>1.214105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target_lag_1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_block_num</th>\n",
       "      <td>28.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>28.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        0          1          2          3          4\n",
       "lgb              0.378975   0.077147   0.107809   0.580430   1.194422\n",
       "xgb2             0.451092   0.100030   0.142697   0.662918   1.272898\n",
       "xgb3             0.477408   0.096444   0.137257   0.686700   1.174996\n",
       "lgb_xgb2        -0.072118  -0.022882  -0.034888  -0.082488  -0.078476\n",
       "lgb_xgb3        -0.098434  -0.019297  -0.029448  -0.106270   0.019426\n",
       "xgb2_xgb3       -0.026316   0.003586   0.005440  -0.023782   0.097902\n",
       "oof_mean         0.435825   0.091207   0.129255   0.643350   1.214105\n",
       "target_lag_1     0.000000   0.000000   0.000000   1.000000   3.000000\n",
       "date_block_num  28.000000  28.000000  28.000000  28.000000  28.000000\n",
       "target           2.000000   1.000000   1.000000   2.000000   2.000000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_oof_df= pd.read_pickle(oof_path+'ensembling_nofast.pickle')\n",
    "all_oof_df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cv = get_cv_idxs(all_oof_df,29,33)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This all should be 20-clipped already\n",
    "lgb_pred = pd.read_csv(submission_path+'coursera_tuned_lightgbm_basic_6folds.csv').item_cnt_month\n",
    "#xgb grow with max leaf node\n",
    "xgb2_pred = pd.read_csv(submission_path+'tuned_xgb_basicfeatures_6folds_8136.csv').item_cnt_month\n",
    "#xgb grow with max depth\n",
    "xgb3_pred = pd.read_csv(submission_path+'tuned_xgb_basicfeatures_6folds_8126.csv').item_cnt_month\n",
    "\n",
    "target_lag_1 = pd.read_csv(data_path+'test_lag.csv').target_lag_1\n",
    "\n",
    "#fastai\n",
    "#???\n",
    "\n",
    "\n",
    "temp = [lgb_pred,xgb2_pred,xgb3_pred,\n",
    "#         fast_oof,\n",
    "        lgb_pred-xgb2_pred,lgb_pred-xgb3_pred,\n",
    "#         lgb_oof-fast_oof,\n",
    "        xgb2_pred-xgb3_pred,\n",
    "#         xgb2_oof-fast_oof,xgb3_oof-fast_oof,\n",
    "        np.mean(np.stack([lgb_pred,xgb2_pred,xgb3_pred],axis=1),axis=1),target_lag_1]\n",
    "\n",
    "temp = np.stack(temp,axis=1)\n",
    "temp.shape\n",
    "\n",
    "test_df = pd.DataFrame(temp,columns=['lgb','xgb2','xgb3',\n",
    "#                                            'fast',\n",
    "                                           'lgb_xgb2','lgb_xgb3',\n",
    "#                                            'lgb_fast',\n",
    "                                           'xgb2_xgb3',\n",
    "#                                            'xgb2_fast','xgb3_fast',\n",
    "                                           'oof_mean','target_lag_1'])\n",
    "\n",
    "test_df.to_csv(data_path+'ensembling_test_nofast.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lgb</th>\n",
       "      <th>xgb2</th>\n",
       "      <th>xgb3</th>\n",
       "      <th>lgb_xgb2</th>\n",
       "      <th>lgb_xgb3</th>\n",
       "      <th>xgb2_xgb3</th>\n",
       "      <th>oof_mean</th>\n",
       "      <th>target_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.596835</td>\n",
       "      <td>0.493997</td>\n",
       "      <td>0.635656</td>\n",
       "      <td>0.102838</td>\n",
       "      <td>-0.038821</td>\n",
       "      <td>-0.141659</td>\n",
       "      <td>0.575496</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.252161</td>\n",
       "      <td>0.248587</td>\n",
       "      <td>0.277100</td>\n",
       "      <td>0.003574</td>\n",
       "      <td>-0.024939</td>\n",
       "      <td>-0.028513</td>\n",
       "      <td>0.259283</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.002808</td>\n",
       "      <td>1.003845</td>\n",
       "      <td>1.193923</td>\n",
       "      <td>-0.001037</td>\n",
       "      <td>-0.191115</td>\n",
       "      <td>-0.190079</td>\n",
       "      <td>1.066859</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.361629</td>\n",
       "      <td>0.311269</td>\n",
       "      <td>0.362728</td>\n",
       "      <td>0.050360</td>\n",
       "      <td>-0.001099</td>\n",
       "      <td>-0.051459</td>\n",
       "      <td>0.345209</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.986176</td>\n",
       "      <td>3.029626</td>\n",
       "      <td>2.800084</td>\n",
       "      <td>-1.043451</td>\n",
       "      <td>-0.813908</td>\n",
       "      <td>0.229542</td>\n",
       "      <td>2.605295</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        lgb      xgb2      xgb3  lgb_xgb2  lgb_xgb3  xgb2_xgb3  oof_mean  \\\n",
       "0  0.596835  0.493997  0.635656  0.102838 -0.038821  -0.141659  0.575496   \n",
       "1  0.252161  0.248587  0.277100  0.003574 -0.024939  -0.028513  0.259283   \n",
       "2  1.002808  1.003845  1.193923 -0.001037 -0.191115  -0.190079  1.066859   \n",
       "3  0.361629  0.311269  0.362728  0.050360 -0.001099  -0.051459  0.345209   \n",
       "4  1.986176  3.029626  2.800084 -1.043451 -0.813908   0.229542  2.605295   \n",
       "\n",
       "   target_lag_1  \n",
       "0           0.0  \n",
       "1           0.0  \n",
       "2           1.0  \n",
       "3           0.0  \n",
       "4           0.0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df= pd.read_csv(data_path+'ensembling_test_nofast.csv')\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# final_pred = lgb_pred*0.8 + xgb2_pred*0.1+xgb3_pred*0.1\n",
    "\n",
    "# get_submission(final_pred,'ensembling_average_8.1.1'); # 0.90876 leaderboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X,y = get_X_y_ensembling(all_oof_df)\n",
    "\n",
    "# elastic = ElasticNetCV(random_state=1402,l1_ratio=[.01,0.05,.1, .5, .7, .9, .95, .99, 1],cv =cv,verbose=10,n_jobs=2)\n",
    "# elastic = ElasticNet(random_state=1402)\n",
    "lr = LinearRegression()\n",
    "params={\n",
    "#     'l1_ratio': [0.0,.001,.005,.01,0.05,.1, .5,.6, .7, .8,.9, 1]\n",
    "    'fit_intercept':[True,False],\n",
    "    'normalize':[True,False]\n",
    "}\n",
    "clf=  GridSearchCV(lr,params,cv=cv,scoring = make_scorer(root_mean_squared_error,greater_is_better=False),verbose=True,n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 4 candidates, totalling 20 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  20 out of  20 | elapsed:   17.5s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=array([[array([     0,      1, ..., 232450, 232451], dtype=int64),\n",
       "        array([232452, 232453, ..., 456738, 456739], dtype=int64)],\n",
       "       [array([     0,      1, ..., 456738, 456739], dtype=int64),\n",
       "        array([456740, 456741, ..., 685627, 685628], dtype=int64)],\n",
       "       [array([     0,     ...dtype=int64),\n",
       "        array([1118820, 1118821, ..., 1356990, 1356991], dtype=int64)]], dtype=object),\n",
       "       error_score='raise',\n",
       "       estimator=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'fit_intercept': [True, False], 'normalize': [True, False]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=make_scorer(root_mean_squared_error, greater_is_better=False),\n",
       "       verbose=True)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'fit_intercept': False, 'normalize': True}\n",
      "-0.806418475542\n"
     ]
    }
   ],
   "source": [
    "print(clf.best_params_)\n",
    "print(clf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean_fit_time</th>\n",
       "      <td>0.291597</td>\n",
       "      <td>0.763952</td>\n",
       "      <td>0.85235</td>\n",
       "      <td>0.85195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_score_time</th>\n",
       "      <td>0.00471992</td>\n",
       "      <td>0.00955987</td>\n",
       "      <td>0.0312397</td>\n",
       "      <td>0.0197998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_test_score</th>\n",
       "      <td>-0.806568</td>\n",
       "      <td>-0.806467</td>\n",
       "      <td>-0.806418</td>\n",
       "      <td>-0.806418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_train_score</th>\n",
       "      <td>-0.789094</td>\n",
       "      <td>-0.789089</td>\n",
       "      <td>-0.789239</td>\n",
       "      <td>-0.789239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_fit_intercept</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_normalize</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>params</th>\n",
       "      <td>{'fit_intercept': True, 'normalize': True}</td>\n",
       "      <td>{'fit_intercept': True, 'normalize': False}</td>\n",
       "      <td>{'fit_intercept': False, 'normalize': True}</td>\n",
       "      <td>{'fit_intercept': False, 'normalize': False}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rank_test_score</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split0_test_score</th>\n",
       "      <td>-0.777867</td>\n",
       "      <td>-0.777262</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.77736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split0_train_score</th>\n",
       "      <td>-0.828671</td>\n",
       "      <td>-0.828645</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split1_test_score</th>\n",
       "      <td>-0.696679</td>\n",
       "      <td>-0.696715</td>\n",
       "      <td>-0.696948</td>\n",
       "      <td>-0.696948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split1_train_score</th>\n",
       "      <td>-0.799446</td>\n",
       "      <td>-0.799446</td>\n",
       "      <td>-0.799593</td>\n",
       "      <td>-0.799593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split2_test_score</th>\n",
       "      <td>-0.760252</td>\n",
       "      <td>-0.76027</td>\n",
       "      <td>-0.760425</td>\n",
       "      <td>-0.760425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split2_train_score</th>\n",
       "      <td>-0.766306</td>\n",
       "      <td>-0.766307</td>\n",
       "      <td>-0.766547</td>\n",
       "      <td>-0.766547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split3_test_score</th>\n",
       "      <td>-0.872432</td>\n",
       "      <td>-0.872457</td>\n",
       "      <td>-0.872141</td>\n",
       "      <td>-0.872141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split3_train_score</th>\n",
       "      <td>-0.764482</td>\n",
       "      <td>-0.764482</td>\n",
       "      <td>-0.76469</td>\n",
       "      <td>-0.76469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split4_test_score</th>\n",
       "      <td>-0.920454</td>\n",
       "      <td>-0.920475</td>\n",
       "      <td>-0.920079</td>\n",
       "      <td>-0.920079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split4_train_score</th>\n",
       "      <td>-0.786564</td>\n",
       "      <td>-0.786564</td>\n",
       "      <td>-0.786677</td>\n",
       "      <td>-0.786677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_fit_time</th>\n",
       "      <td>0.083384</td>\n",
       "      <td>0.59986</td>\n",
       "      <td>0.517021</td>\n",
       "      <td>0.207558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_score_time</th>\n",
       "      <td>0.00626032</td>\n",
       "      <td>0.0123355</td>\n",
       "      <td>0.02209</td>\n",
       "      <td>0.0117764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_test_score</th>\n",
       "      <td>0.081375</td>\n",
       "      <td>0.0814162</td>\n",
       "      <td>0.0811613</td>\n",
       "      <td>0.0811613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_train_score</th>\n",
       "      <td>0.0236849</td>\n",
       "      <td>0.0236764</td>\n",
       "      <td>0.0236112</td>\n",
       "      <td>0.0236112</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                              0  \\\n",
       "mean_fit_time                                          0.291597   \n",
       "mean_score_time                                      0.00471992   \n",
       "mean_test_score                                       -0.806568   \n",
       "mean_train_score                                      -0.789094   \n",
       "param_fit_intercept                                        True   \n",
       "param_normalize                                            True   \n",
       "params               {'fit_intercept': True, 'normalize': True}   \n",
       "rank_test_score                                               4   \n",
       "split0_test_score                                     -0.777867   \n",
       "split0_train_score                                    -0.828671   \n",
       "split1_test_score                                     -0.696679   \n",
       "split1_train_score                                    -0.799446   \n",
       "split2_test_score                                     -0.760252   \n",
       "split2_train_score                                    -0.766306   \n",
       "split3_test_score                                     -0.872432   \n",
       "split3_train_score                                    -0.764482   \n",
       "split4_test_score                                     -0.920454   \n",
       "split4_train_score                                    -0.786564   \n",
       "std_fit_time                                           0.083384   \n",
       "std_score_time                                       0.00626032   \n",
       "std_test_score                                         0.081375   \n",
       "std_train_score                                       0.0236849   \n",
       "\n",
       "                                                               1  \\\n",
       "mean_fit_time                                           0.763952   \n",
       "mean_score_time                                       0.00955987   \n",
       "mean_test_score                                        -0.806467   \n",
       "mean_train_score                                       -0.789089   \n",
       "param_fit_intercept                                         True   \n",
       "param_normalize                                            False   \n",
       "params               {'fit_intercept': True, 'normalize': False}   \n",
       "rank_test_score                                                3   \n",
       "split0_test_score                                      -0.777262   \n",
       "split0_train_score                                     -0.828645   \n",
       "split1_test_score                                      -0.696715   \n",
       "split1_train_score                                     -0.799446   \n",
       "split2_test_score                                       -0.76027   \n",
       "split2_train_score                                     -0.766307   \n",
       "split3_test_score                                      -0.872457   \n",
       "split3_train_score                                     -0.764482   \n",
       "split4_test_score                                      -0.920475   \n",
       "split4_train_score                                     -0.786564   \n",
       "std_fit_time                                             0.59986   \n",
       "std_score_time                                         0.0123355   \n",
       "std_test_score                                         0.0814162   \n",
       "std_train_score                                        0.0236764   \n",
       "\n",
       "                                                               2  \\\n",
       "mean_fit_time                                            0.85235   \n",
       "mean_score_time                                        0.0312397   \n",
       "mean_test_score                                        -0.806418   \n",
       "mean_train_score                                       -0.789239   \n",
       "param_fit_intercept                                        False   \n",
       "param_normalize                                             True   \n",
       "params               {'fit_intercept': False, 'normalize': True}   \n",
       "rank_test_score                                                1   \n",
       "split0_test_score                                       -0.77736   \n",
       "split0_train_score                                     -0.828686   \n",
       "split1_test_score                                      -0.696948   \n",
       "split1_train_score                                     -0.799593   \n",
       "split2_test_score                                      -0.760425   \n",
       "split2_train_score                                     -0.766547   \n",
       "split3_test_score                                      -0.872141   \n",
       "split3_train_score                                      -0.76469   \n",
       "split4_test_score                                      -0.920079   \n",
       "split4_train_score                                     -0.786677   \n",
       "std_fit_time                                            0.517021   \n",
       "std_score_time                                           0.02209   \n",
       "std_test_score                                         0.0811613   \n",
       "std_train_score                                        0.0236112   \n",
       "\n",
       "                                                                3  \n",
       "mean_fit_time                                             0.85195  \n",
       "mean_score_time                                         0.0197998  \n",
       "mean_test_score                                         -0.806418  \n",
       "mean_train_score                                        -0.789239  \n",
       "param_fit_intercept                                         False  \n",
       "param_normalize                                             False  \n",
       "params               {'fit_intercept': False, 'normalize': False}  \n",
       "rank_test_score                                                 1  \n",
       "split0_test_score                                        -0.77736  \n",
       "split0_train_score                                      -0.828686  \n",
       "split1_test_score                                       -0.696948  \n",
       "split1_train_score                                      -0.799593  \n",
       "split2_test_score                                       -0.760425  \n",
       "split2_train_score                                      -0.766547  \n",
       "split3_test_score                                       -0.872141  \n",
       "split3_train_score                                       -0.76469  \n",
       "split4_test_score                                       -0.920079  \n",
       "split4_train_score                                      -0.786677  \n",
       "std_fit_time                                             0.207558  \n",
       "std_score_time                                          0.0117764  \n",
       "std_test_score                                          0.0811613  \n",
       "std_train_score                                         0.0236112  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_result=pd.DataFrame(clf.cv_results_)\n",
    "cv_result.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_l1_ratio</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>...</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split3_train_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>split4_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>37.172250</td>\n",
       "      <td>0.01680</td>\n",
       "      <td>-0.844334</td>\n",
       "      <td>-0.826653</td>\n",
       "      <td>0</td>\n",
       "      <td>{'l1_ratio': 0.0}</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.807423</td>\n",
       "      <td>-0.861509</td>\n",
       "      <td>-0.742919</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.818949</td>\n",
       "      <td>-0.804664</td>\n",
       "      <td>-0.913504</td>\n",
       "      <td>-0.805938</td>\n",
       "      <td>-0.935918</td>\n",
       "      <td>-0.827148</td>\n",
       "      <td>27.141542</td>\n",
       "      <td>0.008103</td>\n",
       "      <td>0.072075</td>\n",
       "      <td>0.020890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.992440</td>\n",
       "      <td>0.01260</td>\n",
       "      <td>-0.844561</td>\n",
       "      <td>-0.826876</td>\n",
       "      <td>0.001</td>\n",
       "      <td>{'l1_ratio': 0.001}</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.807604</td>\n",
       "      <td>-0.861729</td>\n",
       "      <td>-0.743154</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.819214</td>\n",
       "      <td>-0.804886</td>\n",
       "      <td>-0.913784</td>\n",
       "      <td>-0.806171</td>\n",
       "      <td>-0.936101</td>\n",
       "      <td>-0.827380</td>\n",
       "      <td>0.861523</td>\n",
       "      <td>0.006406</td>\n",
       "      <td>0.072073</td>\n",
       "      <td>0.020886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.103959</td>\n",
       "      <td>0.01580</td>\n",
       "      <td>-0.845459</td>\n",
       "      <td>-0.827777</td>\n",
       "      <td>0.005</td>\n",
       "      <td>{'l1_ratio': 0.005}</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.808335</td>\n",
       "      <td>-0.862619</td>\n",
       "      <td>-0.744112</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.820314</td>\n",
       "      <td>-0.805788</td>\n",
       "      <td>-0.914761</td>\n",
       "      <td>-0.807103</td>\n",
       "      <td>-0.936841</td>\n",
       "      <td>-0.828318</td>\n",
       "      <td>0.603955</td>\n",
       "      <td>0.013197</td>\n",
       "      <td>0.072032</td>\n",
       "      <td>0.020872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.109673</td>\n",
       "      <td>0.00972</td>\n",
       "      <td>-0.846567</td>\n",
       "      <td>-0.828914</td>\n",
       "      <td>0.01</td>\n",
       "      <td>{'l1_ratio': 0.01}</td>\n",
       "      <td>4</td>\n",
       "      <td>-0.809259</td>\n",
       "      <td>-0.863749</td>\n",
       "      <td>-0.745388</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.821718</td>\n",
       "      <td>-0.806937</td>\n",
       "      <td>-0.915982</td>\n",
       "      <td>-0.808287</td>\n",
       "      <td>-0.937593</td>\n",
       "      <td>-0.829465</td>\n",
       "      <td>0.583238</td>\n",
       "      <td>0.005761</td>\n",
       "      <td>0.071909</td>\n",
       "      <td>0.020854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.675500</td>\n",
       "      <td>0.00312</td>\n",
       "      <td>-0.856498</td>\n",
       "      <td>-0.838663</td>\n",
       "      <td>0.05</td>\n",
       "      <td>{'l1_ratio': 0.05}</td>\n",
       "      <td>5</td>\n",
       "      <td>-0.819146</td>\n",
       "      <td>-0.872932</td>\n",
       "      <td>-0.756281</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.833623</td>\n",
       "      <td>-0.816919</td>\n",
       "      <td>-0.926504</td>\n",
       "      <td>-0.818564</td>\n",
       "      <td>-0.944320</td>\n",
       "      <td>-0.839425</td>\n",
       "      <td>0.415466</td>\n",
       "      <td>0.006240</td>\n",
       "      <td>0.070773</td>\n",
       "      <td>0.020485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2.265339</td>\n",
       "      <td>0.02636</td>\n",
       "      <td>-0.870917</td>\n",
       "      <td>-0.852805</td>\n",
       "      <td>0.1</td>\n",
       "      <td>{'l1_ratio': 0.1}</td>\n",
       "      <td>6</td>\n",
       "      <td>-0.834158</td>\n",
       "      <td>-0.886029</td>\n",
       "      <td>-0.771680</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.850259</td>\n",
       "      <td>-0.831466</td>\n",
       "      <td>-0.941630</td>\n",
       "      <td>-0.833504</td>\n",
       "      <td>-0.954595</td>\n",
       "      <td>-0.853917</td>\n",
       "      <td>0.978752</td>\n",
       "      <td>0.029936</td>\n",
       "      <td>0.069359</td>\n",
       "      <td>0.019863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.905838</td>\n",
       "      <td>0.03540</td>\n",
       "      <td>-1.038082</td>\n",
       "      <td>-1.022502</td>\n",
       "      <td>0.5</td>\n",
       "      <td>{'l1_ratio': 0.5}</td>\n",
       "      <td>7</td>\n",
       "      <td>-1.017234</td>\n",
       "      <td>-1.057547</td>\n",
       "      <td>-0.946900</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.022024</td>\n",
       "      <td>-0.999357</td>\n",
       "      <td>-1.110829</td>\n",
       "      <td>-1.003214</td>\n",
       "      <td>-1.093021</td>\n",
       "      <td>-1.023134</td>\n",
       "      <td>0.390283</td>\n",
       "      <td>0.037734</td>\n",
       "      <td>0.059130</td>\n",
       "      <td>0.020890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.990000</td>\n",
       "      <td>0.01460</td>\n",
       "      <td>-1.042693</td>\n",
       "      <td>-1.026014</td>\n",
       "      <td>0.6</td>\n",
       "      <td>{'l1_ratio': 0.6}</td>\n",
       "      <td>8</td>\n",
       "      <td>-1.021657</td>\n",
       "      <td>-1.059743</td>\n",
       "      <td>-0.950955</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.026340</td>\n",
       "      <td>-1.002996</td>\n",
       "      <td>-1.115963</td>\n",
       "      <td>-1.007502</td>\n",
       "      <td>-1.098128</td>\n",
       "      <td>-1.027867</td>\n",
       "      <td>0.101398</td>\n",
       "      <td>0.006946</td>\n",
       "      <td>0.059556</td>\n",
       "      <td>0.020244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.056400</td>\n",
       "      <td>0.05648</td>\n",
       "      <td>-1.047587</td>\n",
       "      <td>-1.030137</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'l1_ratio': 0.7}</td>\n",
       "      <td>9</td>\n",
       "      <td>-1.026350</td>\n",
       "      <td>-1.062327</td>\n",
       "      <td>-0.955168</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.030886</td>\n",
       "      <td>-1.007264</td>\n",
       "      <td>-1.121407</td>\n",
       "      <td>-1.012535</td>\n",
       "      <td>-1.103676</td>\n",
       "      <td>-1.033424</td>\n",
       "      <td>0.228955</td>\n",
       "      <td>0.059497</td>\n",
       "      <td>0.060063</td>\n",
       "      <td>0.019519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.189595</td>\n",
       "      <td>0.02332</td>\n",
       "      <td>-1.052774</td>\n",
       "      <td>-1.034889</td>\n",
       "      <td>0.8</td>\n",
       "      <td>{'l1_ratio': 0.8}</td>\n",
       "      <td>10</td>\n",
       "      <td>-1.031316</td>\n",
       "      <td>-1.065305</td>\n",
       "      <td>-0.959543</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.035669</td>\n",
       "      <td>-1.012180</td>\n",
       "      <td>-1.127173</td>\n",
       "      <td>-1.018338</td>\n",
       "      <td>-1.109682</td>\n",
       "      <td>-1.039836</td>\n",
       "      <td>0.378167</td>\n",
       "      <td>0.035289</td>\n",
       "      <td>0.060655</td>\n",
       "      <td>0.018732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.096748</td>\n",
       "      <td>0.00764</td>\n",
       "      <td>-1.058260</td>\n",
       "      <td>-1.040288</td>\n",
       "      <td>0.9</td>\n",
       "      <td>{'l1_ratio': 0.9}</td>\n",
       "      <td>11</td>\n",
       "      <td>-1.036557</td>\n",
       "      <td>-1.068683</td>\n",
       "      <td>-0.964083</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.040694</td>\n",
       "      <td>-1.017760</td>\n",
       "      <td>-1.133270</td>\n",
       "      <td>-1.024934</td>\n",
       "      <td>-1.116162</td>\n",
       "      <td>-1.047133</td>\n",
       "      <td>0.296223</td>\n",
       "      <td>0.006984</td>\n",
       "      <td>0.061336</td>\n",
       "      <td>0.017908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.954232</td>\n",
       "      <td>0.02872</td>\n",
       "      <td>-1.064053</td>\n",
       "      <td>-1.046350</td>\n",
       "      <td>1</td>\n",
       "      <td>{'l1_ratio': 1}</td>\n",
       "      <td>12</td>\n",
       "      <td>-1.042077</td>\n",
       "      <td>-1.072470</td>\n",
       "      <td>-0.968790</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.045966</td>\n",
       "      <td>-1.024020</td>\n",
       "      <td>-1.139707</td>\n",
       "      <td>-1.032346</td>\n",
       "      <td>-1.123134</td>\n",
       "      <td>-1.055346</td>\n",
       "      <td>0.299881</td>\n",
       "      <td>0.023619</td>\n",
       "      <td>0.062113</td>\n",
       "      <td>0.017085</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean_fit_time  mean_score_time  mean_test_score  mean_train_score  \\\n",
       "0       37.172250          0.01680        -0.844334         -0.826653   \n",
       "1        1.992440          0.01260        -0.844561         -0.826876   \n",
       "2        2.103959          0.01580        -0.845459         -0.827777   \n",
       "3        2.109673          0.00972        -0.846567         -0.828914   \n",
       "4        1.675500          0.00312        -0.856498         -0.838663   \n",
       "5        2.265339          0.02636        -0.870917         -0.852805   \n",
       "6        0.905838          0.03540        -1.038082         -1.022502   \n",
       "7        0.990000          0.01460        -1.042693         -1.026014   \n",
       "8        1.056400          0.05648        -1.047587         -1.030137   \n",
       "9        1.189595          0.02332        -1.052774         -1.034889   \n",
       "10       1.096748          0.00764        -1.058260         -1.040288   \n",
       "11       0.954232          0.02872        -1.064053         -1.046350   \n",
       "\n",
       "   param_l1_ratio               params  rank_test_score  split0_test_score  \\\n",
       "0               0    {'l1_ratio': 0.0}                1          -0.807423   \n",
       "1           0.001  {'l1_ratio': 0.001}                2          -0.807604   \n",
       "2           0.005  {'l1_ratio': 0.005}                3          -0.808335   \n",
       "3            0.01   {'l1_ratio': 0.01}                4          -0.809259   \n",
       "4            0.05   {'l1_ratio': 0.05}                5          -0.819146   \n",
       "5             0.1    {'l1_ratio': 0.1}                6          -0.834158   \n",
       "6             0.5    {'l1_ratio': 0.5}                7          -1.017234   \n",
       "7             0.6    {'l1_ratio': 0.6}                8          -1.021657   \n",
       "8             0.7    {'l1_ratio': 0.7}                9          -1.026350   \n",
       "9             0.8    {'l1_ratio': 0.8}               10          -1.031316   \n",
       "10            0.9    {'l1_ratio': 0.9}               11          -1.036557   \n",
       "11              1      {'l1_ratio': 1}               12          -1.042077   \n",
       "\n",
       "    split0_train_score  split1_test_score       ...         split2_test_score  \\\n",
       "0            -0.861509          -0.742919       ...                 -0.818949   \n",
       "1            -0.861729          -0.743154       ...                 -0.819214   \n",
       "2            -0.862619          -0.744112       ...                 -0.820314   \n",
       "3            -0.863749          -0.745388       ...                 -0.821718   \n",
       "4            -0.872932          -0.756281       ...                 -0.833623   \n",
       "5            -0.886029          -0.771680       ...                 -0.850259   \n",
       "6            -1.057547          -0.946900       ...                 -1.022024   \n",
       "7            -1.059743          -0.950955       ...                 -1.026340   \n",
       "8            -1.062327          -0.955168       ...                 -1.030886   \n",
       "9            -1.065305          -0.959543       ...                 -1.035669   \n",
       "10           -1.068683          -0.964083       ...                 -1.040694   \n",
       "11           -1.072470          -0.968790       ...                 -1.045966   \n",
       "\n",
       "    split2_train_score  split3_test_score  split3_train_score  \\\n",
       "0            -0.804664          -0.913504           -0.805938   \n",
       "1            -0.804886          -0.913784           -0.806171   \n",
       "2            -0.805788          -0.914761           -0.807103   \n",
       "3            -0.806937          -0.915982           -0.808287   \n",
       "4            -0.816919          -0.926504           -0.818564   \n",
       "5            -0.831466          -0.941630           -0.833504   \n",
       "6            -0.999357          -1.110829           -1.003214   \n",
       "7            -1.002996          -1.115963           -1.007502   \n",
       "8            -1.007264          -1.121407           -1.012535   \n",
       "9            -1.012180          -1.127173           -1.018338   \n",
       "10           -1.017760          -1.133270           -1.024934   \n",
       "11           -1.024020          -1.139707           -1.032346   \n",
       "\n",
       "    split4_test_score  split4_train_score  std_fit_time  std_score_time  \\\n",
       "0           -0.935918           -0.827148     27.141542        0.008103   \n",
       "1           -0.936101           -0.827380      0.861523        0.006406   \n",
       "2           -0.936841           -0.828318      0.603955        0.013197   \n",
       "3           -0.937593           -0.829465      0.583238        0.005761   \n",
       "4           -0.944320           -0.839425      0.415466        0.006240   \n",
       "5           -0.954595           -0.853917      0.978752        0.029936   \n",
       "6           -1.093021           -1.023134      0.390283        0.037734   \n",
       "7           -1.098128           -1.027867      0.101398        0.006946   \n",
       "8           -1.103676           -1.033424      0.228955        0.059497   \n",
       "9           -1.109682           -1.039836      0.378167        0.035289   \n",
       "10          -1.116162           -1.047133      0.296223        0.006984   \n",
       "11          -1.123134           -1.055346      0.299881        0.023619   \n",
       "\n",
       "    std_test_score  std_train_score  \n",
       "0         0.072075         0.020890  \n",
       "1         0.072073         0.020886  \n",
       "2         0.072032         0.020872  \n",
       "3         0.071909         0.020854  \n",
       "4         0.070773         0.020485  \n",
       "5         0.069359         0.019863  \n",
       "6         0.059130         0.020890  \n",
       "7         0.059556         0.020244  \n",
       "8         0.060063         0.019519  \n",
       "9         0.060655         0.018732  \n",
       "10        0.061336         0.017908  \n",
       "11        0.062113         0.017085  \n",
       "\n",
       "[12 rows x 21 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cv_result=pd.DataFrame(clf.cv_results_)\n",
    "# cv_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X,y = get_X_y_ensembling(all_oof_df)\n",
    "\n",
    "lr = LinearRegression(fit_intercept=False,n_jobs=-1)\n",
    "\n",
    "lr.fit(X,y)\n",
    "\n",
    "X.head()\n",
    "\n",
    "test_pred =  lr.predict(test_df)\n",
    "\n",
    "pd.Series(test_pred).describe()\n",
    "\n",
    "get_submission(test_pred,'ensembling_lr'); # 0.90982 LB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X,y = get_X_y_ensembling(all_oof_df)\n",
    "\n",
    "# elastic = ElasticNetCV(random_state=1402,l1_ratio=[.01,0.05,.1, .5, .7, .9, .95, .99, 1],cv =cv,verbose=10,n_jobs=2)\n",
    "# elastic = ElasticNet(random_state=1402)\n",
    "lr = Ridge(random_state=1402)\n",
    "params={\n",
    "#     'l1_ratio': [0.0,.001,.005,.01,0.05,.1, .5,.6, .7, .8,.9, 1]\n",
    "    \n",
    "    'alpha':[0.0,.001,.005,.01,0.05,.1, .5,.6, .7, .8,.9, 1],\n",
    "    'fit_intercept': [False],\n",
    "    'solver':['svd','cholesky','sparse_cg','lsqr','sag']\n",
    "    \n",
    "}\n",
    "clf=  GridSearchCV(lr,params,cv=cv,scoring = make_scorer(root_mean_squared_error,greater_is_better=False),verbose=True,n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 60 candidates, totalling 300 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:   23.5s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:  3.7min\n",
      "[Parallel(n_jobs=-1)]: Done 300 out of 300 | elapsed:  6.9min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=array([[array([     0,      1, ..., 232450, 232451], dtype=int64),\n",
       "        array([232452, 232453, ..., 456738, 456739], dtype=int64)],\n",
       "       [array([     0,      1, ..., 456738, 456739], dtype=int64),\n",
       "        array([456740, 456741, ..., 685627, 685628], dtype=int64)],\n",
       "       [array([     0,     ...dtype=int64),\n",
       "        array([1118820, 1118821, ..., 1356990, 1356991], dtype=int64)]], dtype=object),\n",
       "       error_score='raise',\n",
       "       estimator=Ridge(alpha=1.0, copy_X=True, fit_intercept=True, max_iter=None,\n",
       "   normalize=False, random_state=1402, solver='auto', tol=0.001),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'alpha': [0.0, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 0.6, 0.7, 0.8, 0.9, 1], 'fit_intercept': [False], 'solver': ['svd', 'cholesky', 'sparse_cg', 'lsqr', 'sag']},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=make_scorer(root_mean_squared_error, greater_is_better=False),\n",
       "       verbose=True)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.0, 'fit_intercept': False, 'solver': 'sag'}\n",
      "-0.806023806956\n"
     ]
    }
   ],
   "source": [
    "print(clf.best_params_)\n",
    "print(clf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>4</th>\n",
       "      <th>9</th>\n",
       "      <th>14</th>\n",
       "      <th>19</th>\n",
       "      <th>24</th>\n",
       "      <th>29</th>\n",
       "      <th>34</th>\n",
       "      <th>39</th>\n",
       "      <th>44</th>\n",
       "      <th>49</th>\n",
       "      <th>...</th>\n",
       "      <th>42</th>\n",
       "      <th>37</th>\n",
       "      <th>32</th>\n",
       "      <th>27</th>\n",
       "      <th>22</th>\n",
       "      <th>17</th>\n",
       "      <th>12</th>\n",
       "      <th>7</th>\n",
       "      <th>2</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean_fit_time</th>\n",
       "      <td>45.8915</td>\n",
       "      <td>44.747</td>\n",
       "      <td>44.9763</td>\n",
       "      <td>45.6628</td>\n",
       "      <td>46.0089</td>\n",
       "      <td>44.8395</td>\n",
       "      <td>44.3174</td>\n",
       "      <td>44.5545</td>\n",
       "      <td>44.9972</td>\n",
       "      <td>45.1884</td>\n",
       "      <td>...</td>\n",
       "      <td>1.07843</td>\n",
       "      <td>1.02219</td>\n",
       "      <td>1.03907</td>\n",
       "      <td>0.885629</td>\n",
       "      <td>1.09659</td>\n",
       "      <td>1.04619</td>\n",
       "      <td>0.922268</td>\n",
       "      <td>1.1824</td>\n",
       "      <td>0.579433</td>\n",
       "      <td>0.243958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_score_time</th>\n",
       "      <td>0.00963993</td>\n",
       "      <td>0.00471992</td>\n",
       "      <td>0.0124798</td>\n",
       "      <td>0.0173999</td>\n",
       "      <td>0.0218397</td>\n",
       "      <td>0.00451994</td>\n",
       "      <td>0.0124799</td>\n",
       "      <td>0.00935984</td>\n",
       "      <td>0.00472002</td>\n",
       "      <td>0.0117599</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0532392</td>\n",
       "      <td>0.0719589</td>\n",
       "      <td>0.0587995</td>\n",
       "      <td>0.0473994</td>\n",
       "      <td>0.0530392</td>\n",
       "      <td>0.0467995</td>\n",
       "      <td>0.0499193</td>\n",
       "      <td>0.0374</td>\n",
       "      <td>0.0343195</td>\n",
       "      <td>0.00803981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_test_score</th>\n",
       "      <td>-0.806024</td>\n",
       "      <td>-0.806024</td>\n",
       "      <td>-0.806024</td>\n",
       "      <td>-0.806024</td>\n",
       "      <td>-0.806024</td>\n",
       "      <td>-0.806024</td>\n",
       "      <td>-0.806024</td>\n",
       "      <td>-0.806024</td>\n",
       "      <td>-0.806024</td>\n",
       "      <td>-0.806024</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.80647</td>\n",
       "      <td>-0.80647</td>\n",
       "      <td>-0.80647</td>\n",
       "      <td>-0.80647</td>\n",
       "      <td>-0.80647</td>\n",
       "      <td>-0.80647</td>\n",
       "      <td>-0.80647</td>\n",
       "      <td>-0.80647</td>\n",
       "      <td>-0.80647</td>\n",
       "      <td>-0.810413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_train_score</th>\n",
       "      <td>-0.789276</td>\n",
       "      <td>-0.789276</td>\n",
       "      <td>-0.789276</td>\n",
       "      <td>-0.789276</td>\n",
       "      <td>-0.789276</td>\n",
       "      <td>-0.789276</td>\n",
       "      <td>-0.789276</td>\n",
       "      <td>-0.789276</td>\n",
       "      <td>-0.789276</td>\n",
       "      <td>-0.789276</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.789243</td>\n",
       "      <td>-0.789243</td>\n",
       "      <td>-0.789243</td>\n",
       "      <td>-0.789243</td>\n",
       "      <td>-0.789243</td>\n",
       "      <td>-0.789243</td>\n",
       "      <td>-0.789243</td>\n",
       "      <td>-0.789243</td>\n",
       "      <td>-0.789243</td>\n",
       "      <td>-0.796948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_alpha</th>\n",
       "      <td>0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.8</td>\n",
       "      <td>...</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_fit_intercept</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_solver</th>\n",
       "      <td>sag</td>\n",
       "      <td>sag</td>\n",
       "      <td>sag</td>\n",
       "      <td>sag</td>\n",
       "      <td>sag</td>\n",
       "      <td>sag</td>\n",
       "      <td>sag</td>\n",
       "      <td>sag</td>\n",
       "      <td>sag</td>\n",
       "      <td>sag</td>\n",
       "      <td>...</td>\n",
       "      <td>sparse_cg</td>\n",
       "      <td>sparse_cg</td>\n",
       "      <td>sparse_cg</td>\n",
       "      <td>sparse_cg</td>\n",
       "      <td>sparse_cg</td>\n",
       "      <td>sparse_cg</td>\n",
       "      <td>sparse_cg</td>\n",
       "      <td>sparse_cg</td>\n",
       "      <td>sparse_cg</td>\n",
       "      <td>svd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>params</th>\n",
       "      <td>{'alpha': 0.0, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.001, 'fit_intercept': False, 'solv...</td>\n",
       "      <td>{'alpha': 0.005, 'fit_intercept': False, 'solv...</td>\n",
       "      <td>{'alpha': 0.01, 'fit_intercept': False, 'solve...</td>\n",
       "      <td>{'alpha': 0.05, 'fit_intercept': False, 'solve...</td>\n",
       "      <td>{'alpha': 0.1, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.5, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.6, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.7, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.8, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>...</td>\n",
       "      <td>{'alpha': 0.7, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.6, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.5, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.1, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.05, 'fit_intercept': False, 'solve...</td>\n",
       "      <td>{'alpha': 0.01, 'fit_intercept': False, 'solve...</td>\n",
       "      <td>{'alpha': 0.005, 'fit_intercept': False, 'solv...</td>\n",
       "      <td>{'alpha': 0.001, 'fit_intercept': False, 'solv...</td>\n",
       "      <td>{'alpha': 0.0, 'fit_intercept': False, 'solver...</td>\n",
       "      <td>{'alpha': 0.0, 'fit_intercept': False, 'solver...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rank_test_score</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>51</td>\n",
       "      <td>52</td>\n",
       "      <td>53</td>\n",
       "      <td>54</td>\n",
       "      <td>55</td>\n",
       "      <td>56</td>\n",
       "      <td>57</td>\n",
       "      <td>58</td>\n",
       "      <td>59</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split0_test_score</th>\n",
       "      <td>-0.774951</td>\n",
       "      <td>-0.774951</td>\n",
       "      <td>-0.774951</td>\n",
       "      <td>-0.774951</td>\n",
       "      <td>-0.774951</td>\n",
       "      <td>-0.774951</td>\n",
       "      <td>-0.774951</td>\n",
       "      <td>-0.774951</td>\n",
       "      <td>-0.774951</td>\n",
       "      <td>-0.774951</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.77736</td>\n",
       "      <td>-0.775314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split0_train_score</th>\n",
       "      <td>-0.828817</td>\n",
       "      <td>-0.828817</td>\n",
       "      <td>-0.828817</td>\n",
       "      <td>-0.828817</td>\n",
       "      <td>-0.828817</td>\n",
       "      <td>-0.828817</td>\n",
       "      <td>-0.828817</td>\n",
       "      <td>-0.828817</td>\n",
       "      <td>-0.828817</td>\n",
       "      <td>-0.828817</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828686</td>\n",
       "      <td>-0.828847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split1_test_score</th>\n",
       "      <td>-0.697159</td>\n",
       "      <td>-0.697159</td>\n",
       "      <td>-0.697159</td>\n",
       "      <td>-0.697159</td>\n",
       "      <td>-0.697159</td>\n",
       "      <td>-0.697159</td>\n",
       "      <td>-0.697159</td>\n",
       "      <td>-0.697159</td>\n",
       "      <td>-0.697159</td>\n",
       "      <td>-0.697159</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.696955</td>\n",
       "      <td>-0.696955</td>\n",
       "      <td>-0.696955</td>\n",
       "      <td>-0.696955</td>\n",
       "      <td>-0.696955</td>\n",
       "      <td>-0.696955</td>\n",
       "      <td>-0.696955</td>\n",
       "      <td>-0.696955</td>\n",
       "      <td>-0.696955</td>\n",
       "      <td>-0.697317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split1_train_score</th>\n",
       "      <td>-0.799621</td>\n",
       "      <td>-0.799621</td>\n",
       "      <td>-0.799621</td>\n",
       "      <td>-0.799621</td>\n",
       "      <td>-0.799621</td>\n",
       "      <td>-0.799621</td>\n",
       "      <td>-0.799621</td>\n",
       "      <td>-0.799621</td>\n",
       "      <td>-0.799621</td>\n",
       "      <td>-0.799621</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.799602</td>\n",
       "      <td>-0.799602</td>\n",
       "      <td>-0.799602</td>\n",
       "      <td>-0.799602</td>\n",
       "      <td>-0.799602</td>\n",
       "      <td>-0.799602</td>\n",
       "      <td>-0.799602</td>\n",
       "      <td>-0.799602</td>\n",
       "      <td>-0.799602</td>\n",
       "      <td>-0.800028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split2_test_score</th>\n",
       "      <td>-0.760274</td>\n",
       "      <td>-0.760274</td>\n",
       "      <td>-0.760274</td>\n",
       "      <td>-0.760274</td>\n",
       "      <td>-0.760274</td>\n",
       "      <td>-0.760274</td>\n",
       "      <td>-0.760274</td>\n",
       "      <td>-0.760274</td>\n",
       "      <td>-0.760274</td>\n",
       "      <td>-0.760274</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.760119</td>\n",
       "      <td>-0.760119</td>\n",
       "      <td>-0.760119</td>\n",
       "      <td>-0.760119</td>\n",
       "      <td>-0.760119</td>\n",
       "      <td>-0.760119</td>\n",
       "      <td>-0.760119</td>\n",
       "      <td>-0.760119</td>\n",
       "      <td>-0.760119</td>\n",
       "      <td>-0.761836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split2_train_score</th>\n",
       "      <td>-0.766561</td>\n",
       "      <td>-0.766561</td>\n",
       "      <td>-0.766561</td>\n",
       "      <td>-0.766561</td>\n",
       "      <td>-0.766561</td>\n",
       "      <td>-0.766561</td>\n",
       "      <td>-0.766561</td>\n",
       "      <td>-0.766561</td>\n",
       "      <td>-0.766561</td>\n",
       "      <td>-0.766561</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.766556</td>\n",
       "      <td>-0.766556</td>\n",
       "      <td>-0.766556</td>\n",
       "      <td>-0.766556</td>\n",
       "      <td>-0.766556</td>\n",
       "      <td>-0.766556</td>\n",
       "      <td>-0.766556</td>\n",
       "      <td>-0.766556</td>\n",
       "      <td>-0.766556</td>\n",
       "      <td>-0.767515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split3_test_score</th>\n",
       "      <td>-0.872232</td>\n",
       "      <td>-0.872232</td>\n",
       "      <td>-0.872232</td>\n",
       "      <td>-0.872232</td>\n",
       "      <td>-0.872232</td>\n",
       "      <td>-0.872232</td>\n",
       "      <td>-0.872232</td>\n",
       "      <td>-0.872232</td>\n",
       "      <td>-0.872232</td>\n",
       "      <td>-0.872232</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.872513</td>\n",
       "      <td>-0.872513</td>\n",
       "      <td>-0.872513</td>\n",
       "      <td>-0.872513</td>\n",
       "      <td>-0.872513</td>\n",
       "      <td>-0.872513</td>\n",
       "      <td>-0.872513</td>\n",
       "      <td>-0.872513</td>\n",
       "      <td>-0.872513</td>\n",
       "      <td>-0.887901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split3_train_score</th>\n",
       "      <td>-0.764701</td>\n",
       "      <td>-0.764701</td>\n",
       "      <td>-0.764701</td>\n",
       "      <td>-0.764701</td>\n",
       "      <td>-0.764701</td>\n",
       "      <td>-0.764701</td>\n",
       "      <td>-0.764701</td>\n",
       "      <td>-0.764701</td>\n",
       "      <td>-0.764701</td>\n",
       "      <td>-0.764701</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.764697</td>\n",
       "      <td>-0.764697</td>\n",
       "      <td>-0.764697</td>\n",
       "      <td>-0.764697</td>\n",
       "      <td>-0.764697</td>\n",
       "      <td>-0.764697</td>\n",
       "      <td>-0.764697</td>\n",
       "      <td>-0.764697</td>\n",
       "      <td>-0.764697</td>\n",
       "      <td>-0.800291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split4_test_score</th>\n",
       "      <td>-0.920334</td>\n",
       "      <td>-0.920334</td>\n",
       "      <td>-0.920334</td>\n",
       "      <td>-0.920334</td>\n",
       "      <td>-0.920334</td>\n",
       "      <td>-0.920334</td>\n",
       "      <td>-0.920334</td>\n",
       "      <td>-0.920334</td>\n",
       "      <td>-0.920334</td>\n",
       "      <td>-0.920334</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.920248</td>\n",
       "      <td>-0.920248</td>\n",
       "      <td>-0.920248</td>\n",
       "      <td>-0.920248</td>\n",
       "      <td>-0.920248</td>\n",
       "      <td>-0.920248</td>\n",
       "      <td>-0.920248</td>\n",
       "      <td>-0.920248</td>\n",
       "      <td>-0.920248</td>\n",
       "      <td>-0.92477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split4_train_score</th>\n",
       "      <td>-0.786683</td>\n",
       "      <td>-0.786683</td>\n",
       "      <td>-0.786683</td>\n",
       "      <td>-0.786683</td>\n",
       "      <td>-0.786683</td>\n",
       "      <td>-0.786683</td>\n",
       "      <td>-0.786683</td>\n",
       "      <td>-0.786683</td>\n",
       "      <td>-0.786683</td>\n",
       "      <td>-0.786683</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.786675</td>\n",
       "      <td>-0.786675</td>\n",
       "      <td>-0.786675</td>\n",
       "      <td>-0.786675</td>\n",
       "      <td>-0.786675</td>\n",
       "      <td>-0.786675</td>\n",
       "      <td>-0.786675</td>\n",
       "      <td>-0.786675</td>\n",
       "      <td>-0.786675</td>\n",
       "      <td>-0.788061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_fit_time</th>\n",
       "      <td>16.0172</td>\n",
       "      <td>16.3949</td>\n",
       "      <td>15.0065</td>\n",
       "      <td>16.22</td>\n",
       "      <td>17.5939</td>\n",
       "      <td>15.235</td>\n",
       "      <td>15.6734</td>\n",
       "      <td>15.3401</td>\n",
       "      <td>15.5937</td>\n",
       "      <td>15.5748</td>\n",
       "      <td>...</td>\n",
       "      <td>0.148148</td>\n",
       "      <td>0.113922</td>\n",
       "      <td>0.24185</td>\n",
       "      <td>0.0815455</td>\n",
       "      <td>0.270483</td>\n",
       "      <td>0.263638</td>\n",
       "      <td>0.0862957</td>\n",
       "      <td>0.582704</td>\n",
       "      <td>0.249388</td>\n",
       "      <td>0.0792727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_score_time</th>\n",
       "      <td>0.00634774</td>\n",
       "      <td>0.00603269</td>\n",
       "      <td>0.0116738</td>\n",
       "      <td>0.0157743</td>\n",
       "      <td>0.0159088</td>\n",
       "      <td>0.00616775</td>\n",
       "      <td>0.00623994</td>\n",
       "      <td>0.00764227</td>\n",
       "      <td>0.00626048</td>\n",
       "      <td>0.006043</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0215442</td>\n",
       "      <td>0.0399976</td>\n",
       "      <td>0.0240001</td>\n",
       "      <td>0.0108398</td>\n",
       "      <td>0.0233477</td>\n",
       "      <td>9.53674e-08</td>\n",
       "      <td>0.00623994</td>\n",
       "      <td>0.00162474</td>\n",
       "      <td>0.0387177</td>\n",
       "      <td>0.00699285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_test_score</th>\n",
       "      <td>0.0813878</td>\n",
       "      <td>0.0813878</td>\n",
       "      <td>0.0813878</td>\n",
       "      <td>0.0813878</td>\n",
       "      <td>0.0813878</td>\n",
       "      <td>0.0813878</td>\n",
       "      <td>0.0813877</td>\n",
       "      <td>0.0813877</td>\n",
       "      <td>0.0813877</td>\n",
       "      <td>0.0813877</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0813012</td>\n",
       "      <td>0.0813012</td>\n",
       "      <td>0.0813012</td>\n",
       "      <td>0.0813012</td>\n",
       "      <td>0.0813012</td>\n",
       "      <td>0.0813012</td>\n",
       "      <td>0.0813012</td>\n",
       "      <td>0.0813012</td>\n",
       "      <td>0.0813012</td>\n",
       "      <td>0.085068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_train_score</th>\n",
       "      <td>0.0236521</td>\n",
       "      <td>0.0236521</td>\n",
       "      <td>0.0236521</td>\n",
       "      <td>0.0236521</td>\n",
       "      <td>0.0236521</td>\n",
       "      <td>0.0236521</td>\n",
       "      <td>0.0236521</td>\n",
       "      <td>0.0236521</td>\n",
       "      <td>0.0236521</td>\n",
       "      <td>0.0236521</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0236089</td>\n",
       "      <td>0.0236089</td>\n",
       "      <td>0.0236089</td>\n",
       "      <td>0.0236089</td>\n",
       "      <td>0.0236089</td>\n",
       "      <td>0.0236089</td>\n",
       "      <td>0.0236089</td>\n",
       "      <td>0.0236089</td>\n",
       "      <td>0.0236089</td>\n",
       "      <td>0.0199171</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    4   \\\n",
       "mean_fit_time                                                  45.8915   \n",
       "mean_score_time                                             0.00963993   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                          0   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.0, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                      1   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                   16.0172   \n",
       "std_score_time                                              0.00634774   \n",
       "std_test_score                                               0.0813878   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                                                    9   \\\n",
       "mean_fit_time                                                   44.747   \n",
       "mean_score_time                                             0.00471992   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                      0.001   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.001, 'fit_intercept': False, 'solv...   \n",
       "rank_test_score                                                      2   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                   16.3949   \n",
       "std_score_time                                              0.00603269   \n",
       "std_test_score                                               0.0813878   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                                                    14  \\\n",
       "mean_fit_time                                                  44.9763   \n",
       "mean_score_time                                              0.0124798   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                      0.005   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.005, 'fit_intercept': False, 'solv...   \n",
       "rank_test_score                                                      3   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                   15.0065   \n",
       "std_score_time                                               0.0116738   \n",
       "std_test_score                                               0.0813878   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                                                    19  \\\n",
       "mean_fit_time                                                  45.6628   \n",
       "mean_score_time                                              0.0173999   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                       0.01   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.01, 'fit_intercept': False, 'solve...   \n",
       "rank_test_score                                                      4   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                     16.22   \n",
       "std_score_time                                               0.0157743   \n",
       "std_test_score                                               0.0813878   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                                                    24  \\\n",
       "mean_fit_time                                                  46.0089   \n",
       "mean_score_time                                              0.0218397   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                       0.05   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.05, 'fit_intercept': False, 'solve...   \n",
       "rank_test_score                                                      5   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                   17.5939   \n",
       "std_score_time                                               0.0159088   \n",
       "std_test_score                                               0.0813878   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                                                    29  \\\n",
       "mean_fit_time                                                  44.8395   \n",
       "mean_score_time                                             0.00451994   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                        0.1   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.1, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                      6   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                    15.235   \n",
       "std_score_time                                              0.00616775   \n",
       "std_test_score                                               0.0813878   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                                                    34  \\\n",
       "mean_fit_time                                                  44.3174   \n",
       "mean_score_time                                              0.0124799   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                        0.5   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.5, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                      7   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                   15.6734   \n",
       "std_score_time                                              0.00623994   \n",
       "std_test_score                                               0.0813877   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                                                    39  \\\n",
       "mean_fit_time                                                  44.5545   \n",
       "mean_score_time                                             0.00935984   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                        0.6   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.6, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                      8   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                   15.3401   \n",
       "std_score_time                                              0.00764227   \n",
       "std_test_score                                               0.0813877   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                                                    44  \\\n",
       "mean_fit_time                                                  44.9972   \n",
       "mean_score_time                                             0.00472002   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                        0.7   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.7, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                      9   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                   15.5937   \n",
       "std_score_time                                              0.00626048   \n",
       "std_test_score                                               0.0813877   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                                                    49  \\\n",
       "mean_fit_time                                                  45.1884   \n",
       "mean_score_time                                              0.0117599   \n",
       "mean_test_score                                              -0.806024   \n",
       "mean_train_score                                             -0.789276   \n",
       "param_alpha                                                        0.8   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                       sag   \n",
       "params               {'alpha': 0.8, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                     10   \n",
       "split0_test_score                                            -0.774951   \n",
       "split0_train_score                                           -0.828817   \n",
       "split1_test_score                                            -0.697159   \n",
       "split1_train_score                                           -0.799621   \n",
       "split2_test_score                                            -0.760274   \n",
       "split2_train_score                                           -0.766561   \n",
       "split3_test_score                                            -0.872232   \n",
       "split3_train_score                                           -0.764701   \n",
       "split4_test_score                                            -0.920334   \n",
       "split4_train_score                                           -0.786683   \n",
       "std_fit_time                                                   15.5748   \n",
       "std_score_time                                                0.006043   \n",
       "std_test_score                                               0.0813877   \n",
       "std_train_score                                              0.0236521   \n",
       "\n",
       "                                           ...                          \\\n",
       "mean_fit_time                              ...                           \n",
       "mean_score_time                            ...                           \n",
       "mean_test_score                            ...                           \n",
       "mean_train_score                           ...                           \n",
       "param_alpha                                ...                           \n",
       "param_fit_intercept                        ...                           \n",
       "param_solver                               ...                           \n",
       "params                                     ...                           \n",
       "rank_test_score                            ...                           \n",
       "split0_test_score                          ...                           \n",
       "split0_train_score                         ...                           \n",
       "split1_test_score                          ...                           \n",
       "split1_train_score                         ...                           \n",
       "split2_test_score                          ...                           \n",
       "split2_train_score                         ...                           \n",
       "split3_test_score                          ...                           \n",
       "split3_train_score                         ...                           \n",
       "split4_test_score                          ...                           \n",
       "split4_train_score                         ...                           \n",
       "std_fit_time                               ...                           \n",
       "std_score_time                             ...                           \n",
       "std_test_score                             ...                           \n",
       "std_train_score                            ...                           \n",
       "\n",
       "                                                                    42  \\\n",
       "mean_fit_time                                                  1.07843   \n",
       "mean_score_time                                              0.0532392   \n",
       "mean_test_score                                               -0.80647   \n",
       "mean_train_score                                             -0.789243   \n",
       "param_alpha                                                        0.7   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                 sparse_cg   \n",
       "params               {'alpha': 0.7, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                     51   \n",
       "split0_test_score                                             -0.77736   \n",
       "split0_train_score                                           -0.828686   \n",
       "split1_test_score                                            -0.696955   \n",
       "split1_train_score                                           -0.799602   \n",
       "split2_test_score                                            -0.760119   \n",
       "split2_train_score                                           -0.766556   \n",
       "split3_test_score                                            -0.872513   \n",
       "split3_train_score                                           -0.764697   \n",
       "split4_test_score                                            -0.920248   \n",
       "split4_train_score                                           -0.786675   \n",
       "std_fit_time                                                  0.148148   \n",
       "std_score_time                                               0.0215442   \n",
       "std_test_score                                               0.0813012   \n",
       "std_train_score                                              0.0236089   \n",
       "\n",
       "                                                                    37  \\\n",
       "mean_fit_time                                                  1.02219   \n",
       "mean_score_time                                              0.0719589   \n",
       "mean_test_score                                               -0.80647   \n",
       "mean_train_score                                             -0.789243   \n",
       "param_alpha                                                        0.6   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                 sparse_cg   \n",
       "params               {'alpha': 0.6, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                     52   \n",
       "split0_test_score                                             -0.77736   \n",
       "split0_train_score                                           -0.828686   \n",
       "split1_test_score                                            -0.696955   \n",
       "split1_train_score                                           -0.799602   \n",
       "split2_test_score                                            -0.760119   \n",
       "split2_train_score                                           -0.766556   \n",
       "split3_test_score                                            -0.872513   \n",
       "split3_train_score                                           -0.764697   \n",
       "split4_test_score                                            -0.920248   \n",
       "split4_train_score                                           -0.786675   \n",
       "std_fit_time                                                  0.113922   \n",
       "std_score_time                                               0.0399976   \n",
       "std_test_score                                               0.0813012   \n",
       "std_train_score                                              0.0236089   \n",
       "\n",
       "                                                                    32  \\\n",
       "mean_fit_time                                                  1.03907   \n",
       "mean_score_time                                              0.0587995   \n",
       "mean_test_score                                               -0.80647   \n",
       "mean_train_score                                             -0.789243   \n",
       "param_alpha                                                        0.5   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                 sparse_cg   \n",
       "params               {'alpha': 0.5, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                     53   \n",
       "split0_test_score                                             -0.77736   \n",
       "split0_train_score                                           -0.828686   \n",
       "split1_test_score                                            -0.696955   \n",
       "split1_train_score                                           -0.799602   \n",
       "split2_test_score                                            -0.760119   \n",
       "split2_train_score                                           -0.766556   \n",
       "split3_test_score                                            -0.872513   \n",
       "split3_train_score                                           -0.764697   \n",
       "split4_test_score                                            -0.920248   \n",
       "split4_train_score                                           -0.786675   \n",
       "std_fit_time                                                   0.24185   \n",
       "std_score_time                                               0.0240001   \n",
       "std_test_score                                               0.0813012   \n",
       "std_train_score                                              0.0236089   \n",
       "\n",
       "                                                                    27  \\\n",
       "mean_fit_time                                                 0.885629   \n",
       "mean_score_time                                              0.0473994   \n",
       "mean_test_score                                               -0.80647   \n",
       "mean_train_score                                             -0.789243   \n",
       "param_alpha                                                        0.1   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                 sparse_cg   \n",
       "params               {'alpha': 0.1, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                     54   \n",
       "split0_test_score                                             -0.77736   \n",
       "split0_train_score                                           -0.828686   \n",
       "split1_test_score                                            -0.696955   \n",
       "split1_train_score                                           -0.799602   \n",
       "split2_test_score                                            -0.760119   \n",
       "split2_train_score                                           -0.766556   \n",
       "split3_test_score                                            -0.872513   \n",
       "split3_train_score                                           -0.764697   \n",
       "split4_test_score                                            -0.920248   \n",
       "split4_train_score                                           -0.786675   \n",
       "std_fit_time                                                 0.0815455   \n",
       "std_score_time                                               0.0108398   \n",
       "std_test_score                                               0.0813012   \n",
       "std_train_score                                              0.0236089   \n",
       "\n",
       "                                                                    22  \\\n",
       "mean_fit_time                                                  1.09659   \n",
       "mean_score_time                                              0.0530392   \n",
       "mean_test_score                                               -0.80647   \n",
       "mean_train_score                                             -0.789243   \n",
       "param_alpha                                                       0.05   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                 sparse_cg   \n",
       "params               {'alpha': 0.05, 'fit_intercept': False, 'solve...   \n",
       "rank_test_score                                                     55   \n",
       "split0_test_score                                             -0.77736   \n",
       "split0_train_score                                           -0.828686   \n",
       "split1_test_score                                            -0.696955   \n",
       "split1_train_score                                           -0.799602   \n",
       "split2_test_score                                            -0.760119   \n",
       "split2_train_score                                           -0.766556   \n",
       "split3_test_score                                            -0.872513   \n",
       "split3_train_score                                           -0.764697   \n",
       "split4_test_score                                            -0.920248   \n",
       "split4_train_score                                           -0.786675   \n",
       "std_fit_time                                                  0.270483   \n",
       "std_score_time                                               0.0233477   \n",
       "std_test_score                                               0.0813012   \n",
       "std_train_score                                              0.0236089   \n",
       "\n",
       "                                                                    17  \\\n",
       "mean_fit_time                                                  1.04619   \n",
       "mean_score_time                                              0.0467995   \n",
       "mean_test_score                                               -0.80647   \n",
       "mean_train_score                                             -0.789243   \n",
       "param_alpha                                                       0.01   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                 sparse_cg   \n",
       "params               {'alpha': 0.01, 'fit_intercept': False, 'solve...   \n",
       "rank_test_score                                                     56   \n",
       "split0_test_score                                             -0.77736   \n",
       "split0_train_score                                           -0.828686   \n",
       "split1_test_score                                            -0.696955   \n",
       "split1_train_score                                           -0.799602   \n",
       "split2_test_score                                            -0.760119   \n",
       "split2_train_score                                           -0.766556   \n",
       "split3_test_score                                            -0.872513   \n",
       "split3_train_score                                           -0.764697   \n",
       "split4_test_score                                            -0.920248   \n",
       "split4_train_score                                           -0.786675   \n",
       "std_fit_time                                                  0.263638   \n",
       "std_score_time                                             9.53674e-08   \n",
       "std_test_score                                               0.0813012   \n",
       "std_train_score                                              0.0236089   \n",
       "\n",
       "                                                                    12  \\\n",
       "mean_fit_time                                                 0.922268   \n",
       "mean_score_time                                              0.0499193   \n",
       "mean_test_score                                               -0.80647   \n",
       "mean_train_score                                             -0.789243   \n",
       "param_alpha                                                      0.005   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                 sparse_cg   \n",
       "params               {'alpha': 0.005, 'fit_intercept': False, 'solv...   \n",
       "rank_test_score                                                     57   \n",
       "split0_test_score                                             -0.77736   \n",
       "split0_train_score                                           -0.828686   \n",
       "split1_test_score                                            -0.696955   \n",
       "split1_train_score                                           -0.799602   \n",
       "split2_test_score                                            -0.760119   \n",
       "split2_train_score                                           -0.766556   \n",
       "split3_test_score                                            -0.872513   \n",
       "split3_train_score                                           -0.764697   \n",
       "split4_test_score                                            -0.920248   \n",
       "split4_train_score                                           -0.786675   \n",
       "std_fit_time                                                 0.0862957   \n",
       "std_score_time                                              0.00623994   \n",
       "std_test_score                                               0.0813012   \n",
       "std_train_score                                              0.0236089   \n",
       "\n",
       "                                                                    7   \\\n",
       "mean_fit_time                                                   1.1824   \n",
       "mean_score_time                                                 0.0374   \n",
       "mean_test_score                                               -0.80647   \n",
       "mean_train_score                                             -0.789243   \n",
       "param_alpha                                                      0.001   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                 sparse_cg   \n",
       "params               {'alpha': 0.001, 'fit_intercept': False, 'solv...   \n",
       "rank_test_score                                                     58   \n",
       "split0_test_score                                             -0.77736   \n",
       "split0_train_score                                           -0.828686   \n",
       "split1_test_score                                            -0.696955   \n",
       "split1_train_score                                           -0.799602   \n",
       "split2_test_score                                            -0.760119   \n",
       "split2_train_score                                           -0.766556   \n",
       "split3_test_score                                            -0.872513   \n",
       "split3_train_score                                           -0.764697   \n",
       "split4_test_score                                            -0.920248   \n",
       "split4_train_score                                           -0.786675   \n",
       "std_fit_time                                                  0.582704   \n",
       "std_score_time                                              0.00162474   \n",
       "std_test_score                                               0.0813012   \n",
       "std_train_score                                              0.0236089   \n",
       "\n",
       "                                                                    2   \\\n",
       "mean_fit_time                                                 0.579433   \n",
       "mean_score_time                                              0.0343195   \n",
       "mean_test_score                                               -0.80647   \n",
       "mean_train_score                                             -0.789243   \n",
       "param_alpha                                                          0   \n",
       "param_fit_intercept                                              False   \n",
       "param_solver                                                 sparse_cg   \n",
       "params               {'alpha': 0.0, 'fit_intercept': False, 'solver...   \n",
       "rank_test_score                                                     59   \n",
       "split0_test_score                                             -0.77736   \n",
       "split0_train_score                                           -0.828686   \n",
       "split1_test_score                                            -0.696955   \n",
       "split1_train_score                                           -0.799602   \n",
       "split2_test_score                                            -0.760119   \n",
       "split2_train_score                                           -0.766556   \n",
       "split3_test_score                                            -0.872513   \n",
       "split3_train_score                                           -0.764697   \n",
       "split4_test_score                                            -0.920248   \n",
       "split4_train_score                                           -0.786675   \n",
       "std_fit_time                                                  0.249388   \n",
       "std_score_time                                               0.0387177   \n",
       "std_test_score                                               0.0813012   \n",
       "std_train_score                                              0.0236089   \n",
       "\n",
       "                                                                    0   \n",
       "mean_fit_time                                                 0.243958  \n",
       "mean_score_time                                             0.00803981  \n",
       "mean_test_score                                              -0.810413  \n",
       "mean_train_score                                             -0.796948  \n",
       "param_alpha                                                          0  \n",
       "param_fit_intercept                                              False  \n",
       "param_solver                                                       svd  \n",
       "params               {'alpha': 0.0, 'fit_intercept': False, 'solver...  \n",
       "rank_test_score                                                     60  \n",
       "split0_test_score                                            -0.775314  \n",
       "split0_train_score                                           -0.828847  \n",
       "split1_test_score                                            -0.697317  \n",
       "split1_train_score                                           -0.800028  \n",
       "split2_test_score                                            -0.761836  \n",
       "split2_train_score                                           -0.767515  \n",
       "split3_test_score                                            -0.887901  \n",
       "split3_train_score                                           -0.800291  \n",
       "split4_test_score                                             -0.92477  \n",
       "split4_train_score                                           -0.788061  \n",
       "std_fit_time                                                 0.0792727  \n",
       "std_score_time                                              0.00699285  \n",
       "std_test_score                                                0.085068  \n",
       "std_train_score                                              0.0199171  \n",
       "\n",
       "[23 rows x 60 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_result=pd.DataFrame(clf.cv_results_).sort_values('rank_test_score')\n",
    "cv_result.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X,y = get_X_y_ensembling(all_oof_df)\n",
    "\n",
    "params={'alpha': 0.0, 'fit_intercept': False, 'solver': 'sag','random_state':1402}\n",
    "lr = Ridge(**params)\n",
    "\n",
    "lr.fit(X,y)\n",
    "\n",
    "test_pred =  lr.predict(test_df)\n",
    "\n",
    "pd.Series(test_pred).describe()\n",
    "\n",
    "get_submission(test_pred,'ensembling_ridge');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Last attempt: Shallow random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X,y = get_X_y_ensembling(all_oof_df)\n",
    "\n",
    "# elastic = ElasticNetCV(random_state=1402,l1_ratio=[.01,0.05,.1, .5, .7, .9, .95, .99, 1],cv =cv,verbose=10,n_jobs=2)\n",
    "# elastic = ElasticNet(random_state=1402)\n",
    "rf = RandomForestRegressor(random_state=1402)\n",
    "params = {'max_depth': [3,4,5,6,7], 'n_estimators': [7,10,15,20,25], 'max_features': [.4,.5,0.7,.9], 'oob_score': [True]}\n",
    "clf=  GridSearchCV(rf,params,cv=cv,scoring = make_scorer(root_mean_squared_error,greater_is_better=False),verbose=True,n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 100 candidates, totalling 500 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:  9.9min\n",
      "[Parallel(n_jobs=-1)]: Done 434 tasks      | elapsed: 30.8min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed: 39.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 5, 'max_features': 0.4, 'n_estimators': 25, 'oob_score': True}\n",
      "-0.807331441099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:724: UserWarning: Some inputs do not have OOB scores. This probably means too few trees were used to compute any reliable oob estimates.\n",
      "  warn(\"Some inputs do not have OOB scores. \"\n"
     ]
    }
   ],
   "source": [
    "clf.fit(X,y)\n",
    "print(clf.best_params_)\n",
    "print(clf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 5, 'max_features': 0.4, 'n_estimators': 25, 'oob_score': True}\n",
      "{'max_depth': 5, 'max_features': 0.4, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 6, 'max_features': 0.4, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 5, 'max_features': 0.7, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 6, 'max_features': 0.4, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 6, 'max_features': 0.4, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 5, 'max_features': 0.5, 'n_estimators': 25, 'oob_score': True}\n",
      "{'max_depth': 6, 'max_features': 0.4, 'n_estimators': 25, 'oob_score': True}\n",
      "{'max_depth': 5, 'max_features': 0.5, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 5, 'max_features': 0.4, 'n_estimators': 15, 'oob_score': True}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    }
   ],
   "source": [
    "for i in pd.DataFrame(clf.cv_results_).sort_values('rank_test_score').params[:10]:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>44</th>\n",
       "      <th>43</th>\n",
       "      <th>61</th>\n",
       "      <th>52</th>\n",
       "      <th>62</th>\n",
       "      <th>60</th>\n",
       "      <th>49</th>\n",
       "      <th>64</th>\n",
       "      <th>48</th>\n",
       "      <th>42</th>\n",
       "      <th>...</th>\n",
       "      <th>75</th>\n",
       "      <th>96</th>\n",
       "      <th>2</th>\n",
       "      <th>91</th>\n",
       "      <th>1</th>\n",
       "      <th>16</th>\n",
       "      <th>90</th>\n",
       "      <th>0</th>\n",
       "      <th>95</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean_fit_time</th>\n",
       "      <td>41.1944</td>\n",
       "      <td>33.2182</td>\n",
       "      <td>18.7152</td>\n",
       "      <td>37.2436</td>\n",
       "      <td>28.5904</td>\n",
       "      <td>13.5068</td>\n",
       "      <td>51.6172</td>\n",
       "      <td>47.19</td>\n",
       "      <td>42.0934</td>\n",
       "      <td>24.5236</td>\n",
       "      <td>...</td>\n",
       "      <td>27.2056</td>\n",
       "      <td>45.3174</td>\n",
       "      <td>15.5134</td>\n",
       "      <td>32.064</td>\n",
       "      <td>9.9432</td>\n",
       "      <td>20.8878</td>\n",
       "      <td>22.7178</td>\n",
       "      <td>4.9206</td>\n",
       "      <td>31.4252</td>\n",
       "      <td>15.401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_score_time</th>\n",
       "      <td>0.3176</td>\n",
       "      <td>0.247119</td>\n",
       "      <td>0.1126</td>\n",
       "      <td>0.153</td>\n",
       "      <td>0.1722</td>\n",
       "      <td>0.0854001</td>\n",
       "      <td>0.2546</td>\n",
       "      <td>0.301</td>\n",
       "      <td>0.2034</td>\n",
       "      <td>0.1594</td>\n",
       "      <td>...</td>\n",
       "      <td>0.16892</td>\n",
       "      <td>0.1358</td>\n",
       "      <td>0.093</td>\n",
       "      <td>0.1338</td>\n",
       "      <td>0.073</td>\n",
       "      <td>0.0678001</td>\n",
       "      <td>0.099</td>\n",
       "      <td>0.041</td>\n",
       "      <td>0.1152</td>\n",
       "      <td>0.0614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_test_score</th>\n",
       "      <td>-0.807331</td>\n",
       "      <td>-0.807551</td>\n",
       "      <td>-0.807643</td>\n",
       "      <td>-0.80776</td>\n",
       "      <td>-0.807813</td>\n",
       "      <td>-0.807822</td>\n",
       "      <td>-0.807828</td>\n",
       "      <td>-0.807867</td>\n",
       "      <td>-0.80789</td>\n",
       "      <td>-0.807921</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.81293</td>\n",
       "      <td>-0.813089</td>\n",
       "      <td>-0.813153</td>\n",
       "      <td>-0.813189</td>\n",
       "      <td>-0.81331</td>\n",
       "      <td>-0.813587</td>\n",
       "      <td>-0.813594</td>\n",
       "      <td>-0.813903</td>\n",
       "      <td>-0.81417</td>\n",
       "      <td>-0.814235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_train_score</th>\n",
       "      <td>-0.770775</td>\n",
       "      <td>-0.770977</td>\n",
       "      <td>-0.763573</td>\n",
       "      <td>-0.769853</td>\n",
       "      <td>-0.76302</td>\n",
       "      <td>-0.764323</td>\n",
       "      <td>-0.77021</td>\n",
       "      <td>-0.762324</td>\n",
       "      <td>-0.770248</td>\n",
       "      <td>-0.77114</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.760494</td>\n",
       "      <td>-0.748664</td>\n",
       "      <td>-0.787206</td>\n",
       "      <td>-0.749744</td>\n",
       "      <td>-0.787189</td>\n",
       "      <td>-0.786328</td>\n",
       "      <td>-0.750645</td>\n",
       "      <td>-0.787873</td>\n",
       "      <td>-0.749389</td>\n",
       "      <td>-0.787111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_max_depth</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_max_features</th>\n",
       "      <td>0.4</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.4</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_n_estimators</th>\n",
       "      <td>25</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>7</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>20</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_oob_score</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>params</th>\n",
       "      <td>{'max_depth': 5, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>{'max_depth': 5, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>{'max_depth': 6, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>{'max_depth': 5, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 6, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>{'max_depth': 6, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>{'max_depth': 5, 'max_features': 0.5, 'n_estim...</td>\n",
       "      <td>{'max_depth': 6, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>{'max_depth': 5, 'max_features': 0.5, 'n_estim...</td>\n",
       "      <td>{'max_depth': 5, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>...</td>\n",
       "      <td>{'max_depth': 6, 'max_features': 0.9, 'n_estim...</td>\n",
       "      <td>{'max_depth': 7, 'max_features': 0.9, 'n_estim...</td>\n",
       "      <td>{'max_depth': 3, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>{'max_depth': 7, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 3, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>{'max_depth': 3, 'max_features': 0.9, 'n_estim...</td>\n",
       "      <td>{'max_depth': 7, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 3, 'max_features': 0.4, 'n_estim...</td>\n",
       "      <td>{'max_depth': 7, 'max_features': 0.9, 'n_estim...</td>\n",
       "      <td>{'max_depth': 3, 'max_features': 0.9, 'n_estim...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rank_test_score</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>91</td>\n",
       "      <td>92</td>\n",
       "      <td>93</td>\n",
       "      <td>94</td>\n",
       "      <td>95</td>\n",
       "      <td>96</td>\n",
       "      <td>97</td>\n",
       "      <td>98</td>\n",
       "      <td>99</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split0_test_score</th>\n",
       "      <td>-0.776382</td>\n",
       "      <td>-0.775812</td>\n",
       "      <td>-0.781937</td>\n",
       "      <td>-0.77619</td>\n",
       "      <td>-0.782138</td>\n",
       "      <td>-0.782227</td>\n",
       "      <td>-0.777836</td>\n",
       "      <td>-0.78131</td>\n",
       "      <td>-0.777788</td>\n",
       "      <td>-0.774963</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.779005</td>\n",
       "      <td>-0.78076</td>\n",
       "      <td>-0.776613</td>\n",
       "      <td>-0.781327</td>\n",
       "      <td>-0.775824</td>\n",
       "      <td>-0.775826</td>\n",
       "      <td>-0.781601</td>\n",
       "      <td>-0.775357</td>\n",
       "      <td>-0.781365</td>\n",
       "      <td>-0.775777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split0_train_score</th>\n",
       "      <td>-0.79405</td>\n",
       "      <td>-0.794067</td>\n",
       "      <td>-0.779411</td>\n",
       "      <td>-0.792775</td>\n",
       "      <td>-0.779155</td>\n",
       "      <td>-0.78021</td>\n",
       "      <td>-0.793461</td>\n",
       "      <td>-0.777883</td>\n",
       "      <td>-0.793345</td>\n",
       "      <td>-0.794812</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.773158</td>\n",
       "      <td>-0.75208</td>\n",
       "      <td>-0.819513</td>\n",
       "      <td>-0.754699</td>\n",
       "      <td>-0.819383</td>\n",
       "      <td>-0.817824</td>\n",
       "      <td>-0.755273</td>\n",
       "      <td>-0.819886</td>\n",
       "      <td>-0.753473</td>\n",
       "      <td>-0.818307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split1_test_score</th>\n",
       "      <td>-0.7007</td>\n",
       "      <td>-0.700505</td>\n",
       "      <td>-0.699379</td>\n",
       "      <td>-0.699637</td>\n",
       "      <td>-0.700277</td>\n",
       "      <td>-0.699881</td>\n",
       "      <td>-0.700367</td>\n",
       "      <td>-0.700531</td>\n",
       "      <td>-0.700581</td>\n",
       "      <td>-0.701161</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.709557</td>\n",
       "      <td>-0.705675</td>\n",
       "      <td>-0.707086</td>\n",
       "      <td>-0.704714</td>\n",
       "      <td>-0.706632</td>\n",
       "      <td>-0.706067</td>\n",
       "      <td>-0.707527</td>\n",
       "      <td>-0.707543</td>\n",
       "      <td>-0.706362</td>\n",
       "      <td>-0.706675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split1_train_score</th>\n",
       "      <td>-0.779479</td>\n",
       "      <td>-0.779506</td>\n",
       "      <td>-0.772524</td>\n",
       "      <td>-0.778557</td>\n",
       "      <td>-0.771615</td>\n",
       "      <td>-0.773976</td>\n",
       "      <td>-0.779446</td>\n",
       "      <td>-0.770298</td>\n",
       "      <td>-0.779526</td>\n",
       "      <td>-0.779661</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.768579</td>\n",
       "      <td>-0.754755</td>\n",
       "      <td>-0.796095</td>\n",
       "      <td>-0.755623</td>\n",
       "      <td>-0.796238</td>\n",
       "      <td>-0.795501</td>\n",
       "      <td>-0.757279</td>\n",
       "      <td>-0.797496</td>\n",
       "      <td>-0.75543</td>\n",
       "      <td>-0.796535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split2_test_score</th>\n",
       "      <td>-0.758577</td>\n",
       "      <td>-0.75866</td>\n",
       "      <td>-0.758225</td>\n",
       "      <td>-0.757989</td>\n",
       "      <td>-0.75828</td>\n",
       "      <td>-0.758181</td>\n",
       "      <td>-0.758073</td>\n",
       "      <td>-0.758207</td>\n",
       "      <td>-0.758183</td>\n",
       "      <td>-0.758814</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.758366</td>\n",
       "      <td>-0.75912</td>\n",
       "      <td>-0.76701</td>\n",
       "      <td>-0.759715</td>\n",
       "      <td>-0.7667</td>\n",
       "      <td>-0.763397</td>\n",
       "      <td>-0.759973</td>\n",
       "      <td>-0.768592</td>\n",
       "      <td>-0.759451</td>\n",
       "      <td>-0.76362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split2_train_score</th>\n",
       "      <td>-0.751873</td>\n",
       "      <td>-0.75207</td>\n",
       "      <td>-0.745677</td>\n",
       "      <td>-0.750967</td>\n",
       "      <td>-0.745199</td>\n",
       "      <td>-0.746459</td>\n",
       "      <td>-0.751038</td>\n",
       "      <td>-0.745049</td>\n",
       "      <td>-0.751263</td>\n",
       "      <td>-0.751875</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.74348</td>\n",
       "      <td>-0.734844</td>\n",
       "      <td>-0.765722</td>\n",
       "      <td>-0.735424</td>\n",
       "      <td>-0.765854</td>\n",
       "      <td>-0.76538</td>\n",
       "      <td>-0.736745</td>\n",
       "      <td>-0.767035</td>\n",
       "      <td>-0.735121</td>\n",
       "      <td>-0.76615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split3_test_score</th>\n",
       "      <td>-0.872699</td>\n",
       "      <td>-0.874178</td>\n",
       "      <td>-0.870727</td>\n",
       "      <td>-0.875375</td>\n",
       "      <td>-0.870421</td>\n",
       "      <td>-0.86971</td>\n",
       "      <td>-0.873128</td>\n",
       "      <td>-0.870111</td>\n",
       "      <td>-0.87328</td>\n",
       "      <td>-0.874882</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.88488</td>\n",
       "      <td>-0.879789</td>\n",
       "      <td>-0.880817</td>\n",
       "      <td>-0.878922</td>\n",
       "      <td>-0.882348</td>\n",
       "      <td>-0.885763</td>\n",
       "      <td>-0.878124</td>\n",
       "      <td>-0.883991</td>\n",
       "      <td>-0.88338</td>\n",
       "      <td>-0.887351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split3_train_score</th>\n",
       "      <td>-0.752969</td>\n",
       "      <td>-0.753326</td>\n",
       "      <td>-0.748474</td>\n",
       "      <td>-0.751999</td>\n",
       "      <td>-0.747695</td>\n",
       "      <td>-0.748829</td>\n",
       "      <td>-0.75194</td>\n",
       "      <td>-0.747468</td>\n",
       "      <td>-0.751951</td>\n",
       "      <td>-0.753497</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.746862</td>\n",
       "      <td>-0.737924</td>\n",
       "      <td>-0.765993</td>\n",
       "      <td>-0.739256</td>\n",
       "      <td>-0.766119</td>\n",
       "      <td>-0.764405</td>\n",
       "      <td>-0.739473</td>\n",
       "      <td>-0.766588</td>\n",
       "      <td>-0.738395</td>\n",
       "      <td>-0.765027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split4_test_score</th>\n",
       "      <td>-0.922858</td>\n",
       "      <td>-0.923187</td>\n",
       "      <td>-0.922494</td>\n",
       "      <td>-0.924154</td>\n",
       "      <td>-0.922474</td>\n",
       "      <td>-0.923554</td>\n",
       "      <td>-0.924212</td>\n",
       "      <td>-0.923616</td>\n",
       "      <td>-0.924107</td>\n",
       "      <td>-0.924315</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.927315</td>\n",
       "      <td>-0.934139</td>\n",
       "      <td>-0.92894</td>\n",
       "      <td>-0.935262</td>\n",
       "      <td>-0.929735</td>\n",
       "      <td>-0.931423</td>\n",
       "      <td>-0.934713</td>\n",
       "      <td>-0.928889</td>\n",
       "      <td>-0.934418</td>\n",
       "      <td>-0.932289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split4_train_score</th>\n",
       "      <td>-0.775505</td>\n",
       "      <td>-0.775915</td>\n",
       "      <td>-0.771779</td>\n",
       "      <td>-0.774967</td>\n",
       "      <td>-0.771435</td>\n",
       "      <td>-0.772139</td>\n",
       "      <td>-0.775164</td>\n",
       "      <td>-0.77092</td>\n",
       "      <td>-0.775155</td>\n",
       "      <td>-0.775855</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.770394</td>\n",
       "      <td>-0.763719</td>\n",
       "      <td>-0.788707</td>\n",
       "      <td>-0.76372</td>\n",
       "      <td>-0.78835</td>\n",
       "      <td>-0.78853</td>\n",
       "      <td>-0.764455</td>\n",
       "      <td>-0.78836</td>\n",
       "      <td>-0.764525</td>\n",
       "      <td>-0.789538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_fit_time</th>\n",
       "      <td>20.5057</td>\n",
       "      <td>16.411</td>\n",
       "      <td>9.69843</td>\n",
       "      <td>17.7478</td>\n",
       "      <td>14.806</td>\n",
       "      <td>6.9785</td>\n",
       "      <td>26.0035</td>\n",
       "      <td>23.3089</td>\n",
       "      <td>21.5374</td>\n",
       "      <td>11.9635</td>\n",
       "      <td>...</td>\n",
       "      <td>13.4075</td>\n",
       "      <td>22.9811</td>\n",
       "      <td>7.91747</td>\n",
       "      <td>16.3832</td>\n",
       "      <td>5.65951</td>\n",
       "      <td>10.3422</td>\n",
       "      <td>11.4895</td>\n",
       "      <td>2.84534</td>\n",
       "      <td>15.8764</td>\n",
       "      <td>7.81093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_score_time</th>\n",
       "      <td>0.0930476</td>\n",
       "      <td>0.0723981</td>\n",
       "      <td>0.00778714</td>\n",
       "      <td>0.00742962</td>\n",
       "      <td>0.00984689</td>\n",
       "      <td>0.00628015</td>\n",
       "      <td>0.0265374</td>\n",
       "      <td>0.0435063</td>\n",
       "      <td>0.0155641</td>\n",
       "      <td>0.0275072</td>\n",
       "      <td>...</td>\n",
       "      <td>0.116626</td>\n",
       "      <td>0.0115134</td>\n",
       "      <td>0.00787402</td>\n",
       "      <td>0.00872701</td>\n",
       "      <td>0.0231518</td>\n",
       "      <td>0.00466475</td>\n",
       "      <td>0.00689927</td>\n",
       "      <td>0.00807464</td>\n",
       "      <td>0.0322328</td>\n",
       "      <td>0.0159824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_test_score</th>\n",
       "      <td>0.081341</td>\n",
       "      <td>0.0817586</td>\n",
       "      <td>0.080938</td>\n",
       "      <td>0.0825174</td>\n",
       "      <td>0.0806221</td>\n",
       "      <td>0.0809553</td>\n",
       "      <td>0.081854</td>\n",
       "      <td>0.0809139</td>\n",
       "      <td>0.0817795</td>\n",
       "      <td>0.0820838</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0821599</td>\n",
       "      <td>0.0841489</td>\n",
       "      <td>0.0818049</td>\n",
       "      <td>0.0844944</td>\n",
       "      <td>0.0825144</td>\n",
       "      <td>0.0840937</td>\n",
       "      <td>0.0834218</td>\n",
       "      <td>0.0821342</td>\n",
       "      <td>0.0845303</td>\n",
       "      <td>0.0844375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_train_score</th>\n",
       "      <td>0.0162125</td>\n",
       "      <td>0.0161202</td>\n",
       "      <td>0.0137588</td>\n",
       "      <td>0.0161418</td>\n",
       "      <td>0.0138381</td>\n",
       "      <td>0.0138985</td>\n",
       "      <td>0.0164425</td>\n",
       "      <td>0.0134065</td>\n",
       "      <td>0.0163635</td>\n",
       "      <td>0.0163563</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0126415</td>\n",
       "      <td>0.0107866</td>\n",
       "      <td>0.0201821</td>\n",
       "      <td>0.0106724</td>\n",
       "      <td>0.0200938</td>\n",
       "      <td>0.020002</td>\n",
       "      <td>0.010716</td>\n",
       "      <td>0.0200245</td>\n",
       "      <td>0.0110156</td>\n",
       "      <td>0.0199749</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                   44  \\\n",
       "mean_fit_time                                                 41.1944   \n",
       "mean_score_time                                                0.3176   \n",
       "mean_test_score                                             -0.807331   \n",
       "mean_train_score                                            -0.770775   \n",
       "param_max_depth                                                     5   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                 25   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 5, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                     1   \n",
       "split0_test_score                                           -0.776382   \n",
       "split0_train_score                                           -0.79405   \n",
       "split1_test_score                                             -0.7007   \n",
       "split1_train_score                                          -0.779479   \n",
       "split2_test_score                                           -0.758577   \n",
       "split2_train_score                                          -0.751873   \n",
       "split3_test_score                                           -0.872699   \n",
       "split3_train_score                                          -0.752969   \n",
       "split4_test_score                                           -0.922858   \n",
       "split4_train_score                                          -0.775505   \n",
       "std_fit_time                                                  20.5057   \n",
       "std_score_time                                              0.0930476   \n",
       "std_test_score                                               0.081341   \n",
       "std_train_score                                             0.0162125   \n",
       "\n",
       "                                                                   43  \\\n",
       "mean_fit_time                                                 33.2182   \n",
       "mean_score_time                                              0.247119   \n",
       "mean_test_score                                             -0.807551   \n",
       "mean_train_score                                            -0.770977   \n",
       "param_max_depth                                                     5   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                 20   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 5, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                     2   \n",
       "split0_test_score                                           -0.775812   \n",
       "split0_train_score                                          -0.794067   \n",
       "split1_test_score                                           -0.700505   \n",
       "split1_train_score                                          -0.779506   \n",
       "split2_test_score                                            -0.75866   \n",
       "split2_train_score                                           -0.75207   \n",
       "split3_test_score                                           -0.874178   \n",
       "split3_train_score                                          -0.753326   \n",
       "split4_test_score                                           -0.923187   \n",
       "split4_train_score                                          -0.775915   \n",
       "std_fit_time                                                   16.411   \n",
       "std_score_time                                              0.0723981   \n",
       "std_test_score                                              0.0817586   \n",
       "std_train_score                                             0.0161202   \n",
       "\n",
       "                                                                   61  \\\n",
       "mean_fit_time                                                 18.7152   \n",
       "mean_score_time                                                0.1126   \n",
       "mean_test_score                                             -0.807643   \n",
       "mean_train_score                                            -0.763573   \n",
       "param_max_depth                                                     6   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                 10   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 6, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                     3   \n",
       "split0_test_score                                           -0.781937   \n",
       "split0_train_score                                          -0.779411   \n",
       "split1_test_score                                           -0.699379   \n",
       "split1_train_score                                          -0.772524   \n",
       "split2_test_score                                           -0.758225   \n",
       "split2_train_score                                          -0.745677   \n",
       "split3_test_score                                           -0.870727   \n",
       "split3_train_score                                          -0.748474   \n",
       "split4_test_score                                           -0.922494   \n",
       "split4_train_score                                          -0.771779   \n",
       "std_fit_time                                                  9.69843   \n",
       "std_score_time                                             0.00778714   \n",
       "std_test_score                                               0.080938   \n",
       "std_train_score                                             0.0137588   \n",
       "\n",
       "                                                                   52  \\\n",
       "mean_fit_time                                                 37.2436   \n",
       "mean_score_time                                                 0.153   \n",
       "mean_test_score                                              -0.80776   \n",
       "mean_train_score                                            -0.769853   \n",
       "param_max_depth                                                     5   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                 15   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 5, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                     4   \n",
       "split0_test_score                                            -0.77619   \n",
       "split0_train_score                                          -0.792775   \n",
       "split1_test_score                                           -0.699637   \n",
       "split1_train_score                                          -0.778557   \n",
       "split2_test_score                                           -0.757989   \n",
       "split2_train_score                                          -0.750967   \n",
       "split3_test_score                                           -0.875375   \n",
       "split3_train_score                                          -0.751999   \n",
       "split4_test_score                                           -0.924154   \n",
       "split4_train_score                                          -0.774967   \n",
       "std_fit_time                                                  17.7478   \n",
       "std_score_time                                             0.00742962   \n",
       "std_test_score                                              0.0825174   \n",
       "std_train_score                                             0.0161418   \n",
       "\n",
       "                                                                   62  \\\n",
       "mean_fit_time                                                 28.5904   \n",
       "mean_score_time                                                0.1722   \n",
       "mean_test_score                                             -0.807813   \n",
       "mean_train_score                                             -0.76302   \n",
       "param_max_depth                                                     6   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                 15   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 6, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                     5   \n",
       "split0_test_score                                           -0.782138   \n",
       "split0_train_score                                          -0.779155   \n",
       "split1_test_score                                           -0.700277   \n",
       "split1_train_score                                          -0.771615   \n",
       "split2_test_score                                            -0.75828   \n",
       "split2_train_score                                          -0.745199   \n",
       "split3_test_score                                           -0.870421   \n",
       "split3_train_score                                          -0.747695   \n",
       "split4_test_score                                           -0.922474   \n",
       "split4_train_score                                          -0.771435   \n",
       "std_fit_time                                                   14.806   \n",
       "std_score_time                                             0.00984689   \n",
       "std_test_score                                              0.0806221   \n",
       "std_train_score                                             0.0138381   \n",
       "\n",
       "                                                                   60  \\\n",
       "mean_fit_time                                                 13.5068   \n",
       "mean_score_time                                             0.0854001   \n",
       "mean_test_score                                             -0.807822   \n",
       "mean_train_score                                            -0.764323   \n",
       "param_max_depth                                                     6   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                  7   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 6, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                     6   \n",
       "split0_test_score                                           -0.782227   \n",
       "split0_train_score                                           -0.78021   \n",
       "split1_test_score                                           -0.699881   \n",
       "split1_train_score                                          -0.773976   \n",
       "split2_test_score                                           -0.758181   \n",
       "split2_train_score                                          -0.746459   \n",
       "split3_test_score                                            -0.86971   \n",
       "split3_train_score                                          -0.748829   \n",
       "split4_test_score                                           -0.923554   \n",
       "split4_train_score                                          -0.772139   \n",
       "std_fit_time                                                   6.9785   \n",
       "std_score_time                                             0.00628015   \n",
       "std_test_score                                              0.0809553   \n",
       "std_train_score                                             0.0138985   \n",
       "\n",
       "                                                                   49  \\\n",
       "mean_fit_time                                                 51.6172   \n",
       "mean_score_time                                                0.2546   \n",
       "mean_test_score                                             -0.807828   \n",
       "mean_train_score                                             -0.77021   \n",
       "param_max_depth                                                     5   \n",
       "param_max_features                                                0.5   \n",
       "param_n_estimators                                                 25   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 5, 'max_features': 0.5, 'n_estim...   \n",
       "rank_test_score                                                     7   \n",
       "split0_test_score                                           -0.777836   \n",
       "split0_train_score                                          -0.793461   \n",
       "split1_test_score                                           -0.700367   \n",
       "split1_train_score                                          -0.779446   \n",
       "split2_test_score                                           -0.758073   \n",
       "split2_train_score                                          -0.751038   \n",
       "split3_test_score                                           -0.873128   \n",
       "split3_train_score                                           -0.75194   \n",
       "split4_test_score                                           -0.924212   \n",
       "split4_train_score                                          -0.775164   \n",
       "std_fit_time                                                  26.0035   \n",
       "std_score_time                                              0.0265374   \n",
       "std_test_score                                               0.081854   \n",
       "std_train_score                                             0.0164425   \n",
       "\n",
       "                                                                   64  \\\n",
       "mean_fit_time                                                   47.19   \n",
       "mean_score_time                                                 0.301   \n",
       "mean_test_score                                             -0.807867   \n",
       "mean_train_score                                            -0.762324   \n",
       "param_max_depth                                                     6   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                 25   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 6, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                     8   \n",
       "split0_test_score                                            -0.78131   \n",
       "split0_train_score                                          -0.777883   \n",
       "split1_test_score                                           -0.700531   \n",
       "split1_train_score                                          -0.770298   \n",
       "split2_test_score                                           -0.758207   \n",
       "split2_train_score                                          -0.745049   \n",
       "split3_test_score                                           -0.870111   \n",
       "split3_train_score                                          -0.747468   \n",
       "split4_test_score                                           -0.923616   \n",
       "split4_train_score                                           -0.77092   \n",
       "std_fit_time                                                  23.3089   \n",
       "std_score_time                                              0.0435063   \n",
       "std_test_score                                              0.0809139   \n",
       "std_train_score                                             0.0134065   \n",
       "\n",
       "                                                                   48  \\\n",
       "mean_fit_time                                                 42.0934   \n",
       "mean_score_time                                                0.2034   \n",
       "mean_test_score                                              -0.80789   \n",
       "mean_train_score                                            -0.770248   \n",
       "param_max_depth                                                     5   \n",
       "param_max_features                                                0.5   \n",
       "param_n_estimators                                                 20   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 5, 'max_features': 0.5, 'n_estim...   \n",
       "rank_test_score                                                     9   \n",
       "split0_test_score                                           -0.777788   \n",
       "split0_train_score                                          -0.793345   \n",
       "split1_test_score                                           -0.700581   \n",
       "split1_train_score                                          -0.779526   \n",
       "split2_test_score                                           -0.758183   \n",
       "split2_train_score                                          -0.751263   \n",
       "split3_test_score                                            -0.87328   \n",
       "split3_train_score                                          -0.751951   \n",
       "split4_test_score                                           -0.924107   \n",
       "split4_train_score                                          -0.775155   \n",
       "std_fit_time                                                  21.5374   \n",
       "std_score_time                                              0.0155641   \n",
       "std_test_score                                              0.0817795   \n",
       "std_train_score                                             0.0163635   \n",
       "\n",
       "                                                                   42  \\\n",
       "mean_fit_time                                                 24.5236   \n",
       "mean_score_time                                                0.1594   \n",
       "mean_test_score                                             -0.807921   \n",
       "mean_train_score                                             -0.77114   \n",
       "param_max_depth                                                     5   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                 15   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 5, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                    10   \n",
       "split0_test_score                                           -0.774963   \n",
       "split0_train_score                                          -0.794812   \n",
       "split1_test_score                                           -0.701161   \n",
       "split1_train_score                                          -0.779661   \n",
       "split2_test_score                                           -0.758814   \n",
       "split2_train_score                                          -0.751875   \n",
       "split3_test_score                                           -0.874882   \n",
       "split3_train_score                                          -0.753497   \n",
       "split4_test_score                                           -0.924315   \n",
       "split4_train_score                                          -0.775855   \n",
       "std_fit_time                                                  11.9635   \n",
       "std_score_time                                              0.0275072   \n",
       "std_test_score                                              0.0820838   \n",
       "std_train_score                                             0.0163563   \n",
       "\n",
       "                                          ...                          \\\n",
       "mean_fit_time                             ...                           \n",
       "mean_score_time                           ...                           \n",
       "mean_test_score                           ...                           \n",
       "mean_train_score                          ...                           \n",
       "param_max_depth                           ...                           \n",
       "param_max_features                        ...                           \n",
       "param_n_estimators                        ...                           \n",
       "param_oob_score                           ...                           \n",
       "params                                    ...                           \n",
       "rank_test_score                           ...                           \n",
       "split0_test_score                         ...                           \n",
       "split0_train_score                        ...                           \n",
       "split1_test_score                         ...                           \n",
       "split1_train_score                        ...                           \n",
       "split2_test_score                         ...                           \n",
       "split2_train_score                        ...                           \n",
       "split3_test_score                         ...                           \n",
       "split3_train_score                        ...                           \n",
       "split4_test_score                         ...                           \n",
       "split4_train_score                        ...                           \n",
       "std_fit_time                              ...                           \n",
       "std_score_time                            ...                           \n",
       "std_test_score                            ...                           \n",
       "std_train_score                           ...                           \n",
       "\n",
       "                                                                   75  \\\n",
       "mean_fit_time                                                 27.2056   \n",
       "mean_score_time                                               0.16892   \n",
       "mean_test_score                                              -0.81293   \n",
       "mean_train_score                                            -0.760494   \n",
       "param_max_depth                                                     6   \n",
       "param_max_features                                                0.9   \n",
       "param_n_estimators                                                  7   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 6, 'max_features': 0.9, 'n_estim...   \n",
       "rank_test_score                                                    91   \n",
       "split0_test_score                                           -0.779005   \n",
       "split0_train_score                                          -0.773158   \n",
       "split1_test_score                                           -0.709557   \n",
       "split1_train_score                                          -0.768579   \n",
       "split2_test_score                                           -0.758366   \n",
       "split2_train_score                                           -0.74348   \n",
       "split3_test_score                                            -0.88488   \n",
       "split3_train_score                                          -0.746862   \n",
       "split4_test_score                                           -0.927315   \n",
       "split4_train_score                                          -0.770394   \n",
       "std_fit_time                                                  13.4075   \n",
       "std_score_time                                               0.116626   \n",
       "std_test_score                                              0.0821599   \n",
       "std_train_score                                             0.0126415   \n",
       "\n",
       "                                                                   96  \\\n",
       "mean_fit_time                                                 45.3174   \n",
       "mean_score_time                                                0.1358   \n",
       "mean_test_score                                             -0.813089   \n",
       "mean_train_score                                            -0.748664   \n",
       "param_max_depth                                                     7   \n",
       "param_max_features                                                0.9   \n",
       "param_n_estimators                                                 10   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 7, 'max_features': 0.9, 'n_estim...   \n",
       "rank_test_score                                                    92   \n",
       "split0_test_score                                            -0.78076   \n",
       "split0_train_score                                           -0.75208   \n",
       "split1_test_score                                           -0.705675   \n",
       "split1_train_score                                          -0.754755   \n",
       "split2_test_score                                            -0.75912   \n",
       "split2_train_score                                          -0.734844   \n",
       "split3_test_score                                           -0.879789   \n",
       "split3_train_score                                          -0.737924   \n",
       "split4_test_score                                           -0.934139   \n",
       "split4_train_score                                          -0.763719   \n",
       "std_fit_time                                                  22.9811   \n",
       "std_score_time                                              0.0115134   \n",
       "std_test_score                                              0.0841489   \n",
       "std_train_score                                             0.0107866   \n",
       "\n",
       "                                                                   2   \\\n",
       "mean_fit_time                                                 15.5134   \n",
       "mean_score_time                                                 0.093   \n",
       "mean_test_score                                             -0.813153   \n",
       "mean_train_score                                            -0.787206   \n",
       "param_max_depth                                                     3   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                 15   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 3, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                    93   \n",
       "split0_test_score                                           -0.776613   \n",
       "split0_train_score                                          -0.819513   \n",
       "split1_test_score                                           -0.707086   \n",
       "split1_train_score                                          -0.796095   \n",
       "split2_test_score                                            -0.76701   \n",
       "split2_train_score                                          -0.765722   \n",
       "split3_test_score                                           -0.880817   \n",
       "split3_train_score                                          -0.765993   \n",
       "split4_test_score                                            -0.92894   \n",
       "split4_train_score                                          -0.788707   \n",
       "std_fit_time                                                  7.91747   \n",
       "std_score_time                                             0.00787402   \n",
       "std_test_score                                              0.0818049   \n",
       "std_train_score                                             0.0201821   \n",
       "\n",
       "                                                                   91  \\\n",
       "mean_fit_time                                                  32.064   \n",
       "mean_score_time                                                0.1338   \n",
       "mean_test_score                                             -0.813189   \n",
       "mean_train_score                                            -0.749744   \n",
       "param_max_depth                                                     7   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                 10   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 7, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                    94   \n",
       "split0_test_score                                           -0.781327   \n",
       "split0_train_score                                          -0.754699   \n",
       "split1_test_score                                           -0.704714   \n",
       "split1_train_score                                          -0.755623   \n",
       "split2_test_score                                           -0.759715   \n",
       "split2_train_score                                          -0.735424   \n",
       "split3_test_score                                           -0.878922   \n",
       "split3_train_score                                          -0.739256   \n",
       "split4_test_score                                           -0.935262   \n",
       "split4_train_score                                           -0.76372   \n",
       "std_fit_time                                                  16.3832   \n",
       "std_score_time                                             0.00872701   \n",
       "std_test_score                                              0.0844944   \n",
       "std_train_score                                             0.0106724   \n",
       "\n",
       "                                                                   1   \\\n",
       "mean_fit_time                                                  9.9432   \n",
       "mean_score_time                                                 0.073   \n",
       "mean_test_score                                              -0.81331   \n",
       "mean_train_score                                            -0.787189   \n",
       "param_max_depth                                                     3   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                 10   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 3, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                    95   \n",
       "split0_test_score                                           -0.775824   \n",
       "split0_train_score                                          -0.819383   \n",
       "split1_test_score                                           -0.706632   \n",
       "split1_train_score                                          -0.796238   \n",
       "split2_test_score                                             -0.7667   \n",
       "split2_train_score                                          -0.765854   \n",
       "split3_test_score                                           -0.882348   \n",
       "split3_train_score                                          -0.766119   \n",
       "split4_test_score                                           -0.929735   \n",
       "split4_train_score                                           -0.78835   \n",
       "std_fit_time                                                  5.65951   \n",
       "std_score_time                                              0.0231518   \n",
       "std_test_score                                              0.0825144   \n",
       "std_train_score                                             0.0200938   \n",
       "\n",
       "                                                                   16  \\\n",
       "mean_fit_time                                                 20.8878   \n",
       "mean_score_time                                             0.0678001   \n",
       "mean_test_score                                             -0.813587   \n",
       "mean_train_score                                            -0.786328   \n",
       "param_max_depth                                                     3   \n",
       "param_max_features                                                0.9   \n",
       "param_n_estimators                                                 10   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 3, 'max_features': 0.9, 'n_estim...   \n",
       "rank_test_score                                                    96   \n",
       "split0_test_score                                           -0.775826   \n",
       "split0_train_score                                          -0.817824   \n",
       "split1_test_score                                           -0.706067   \n",
       "split1_train_score                                          -0.795501   \n",
       "split2_test_score                                           -0.763397   \n",
       "split2_train_score                                           -0.76538   \n",
       "split3_test_score                                           -0.885763   \n",
       "split3_train_score                                          -0.764405   \n",
       "split4_test_score                                           -0.931423   \n",
       "split4_train_score                                           -0.78853   \n",
       "std_fit_time                                                  10.3422   \n",
       "std_score_time                                             0.00466475   \n",
       "std_test_score                                              0.0840937   \n",
       "std_train_score                                              0.020002   \n",
       "\n",
       "                                                                   90  \\\n",
       "mean_fit_time                                                 22.7178   \n",
       "mean_score_time                                                 0.099   \n",
       "mean_test_score                                             -0.813594   \n",
       "mean_train_score                                            -0.750645   \n",
       "param_max_depth                                                     7   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                  7   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 7, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                    97   \n",
       "split0_test_score                                           -0.781601   \n",
       "split0_train_score                                          -0.755273   \n",
       "split1_test_score                                           -0.707527   \n",
       "split1_train_score                                          -0.757279   \n",
       "split2_test_score                                           -0.759973   \n",
       "split2_train_score                                          -0.736745   \n",
       "split3_test_score                                           -0.878124   \n",
       "split3_train_score                                          -0.739473   \n",
       "split4_test_score                                           -0.934713   \n",
       "split4_train_score                                          -0.764455   \n",
       "std_fit_time                                                  11.4895   \n",
       "std_score_time                                             0.00689927   \n",
       "std_test_score                                              0.0834218   \n",
       "std_train_score                                              0.010716   \n",
       "\n",
       "                                                                   0   \\\n",
       "mean_fit_time                                                  4.9206   \n",
       "mean_score_time                                                 0.041   \n",
       "mean_test_score                                             -0.813903   \n",
       "mean_train_score                                            -0.787873   \n",
       "param_max_depth                                                     3   \n",
       "param_max_features                                                0.4   \n",
       "param_n_estimators                                                  7   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 3, 'max_features': 0.4, 'n_estim...   \n",
       "rank_test_score                                                    98   \n",
       "split0_test_score                                           -0.775357   \n",
       "split0_train_score                                          -0.819886   \n",
       "split1_test_score                                           -0.707543   \n",
       "split1_train_score                                          -0.797496   \n",
       "split2_test_score                                           -0.768592   \n",
       "split2_train_score                                          -0.767035   \n",
       "split3_test_score                                           -0.883991   \n",
       "split3_train_score                                          -0.766588   \n",
       "split4_test_score                                           -0.928889   \n",
       "split4_train_score                                           -0.78836   \n",
       "std_fit_time                                                  2.84534   \n",
       "std_score_time                                             0.00807464   \n",
       "std_test_score                                              0.0821342   \n",
       "std_train_score                                             0.0200245   \n",
       "\n",
       "                                                                   95  \\\n",
       "mean_fit_time                                                 31.4252   \n",
       "mean_score_time                                                0.1152   \n",
       "mean_test_score                                              -0.81417   \n",
       "mean_train_score                                            -0.749389   \n",
       "param_max_depth                                                     7   \n",
       "param_max_features                                                0.9   \n",
       "param_n_estimators                                                  7   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 7, 'max_features': 0.9, 'n_estim...   \n",
       "rank_test_score                                                    99   \n",
       "split0_test_score                                           -0.781365   \n",
       "split0_train_score                                          -0.753473   \n",
       "split1_test_score                                           -0.706362   \n",
       "split1_train_score                                           -0.75543   \n",
       "split2_test_score                                           -0.759451   \n",
       "split2_train_score                                          -0.735121   \n",
       "split3_test_score                                            -0.88338   \n",
       "split3_train_score                                          -0.738395   \n",
       "split4_test_score                                           -0.934418   \n",
       "split4_train_score                                          -0.764525   \n",
       "std_fit_time                                                  15.8764   \n",
       "std_score_time                                              0.0322328   \n",
       "std_test_score                                              0.0845303   \n",
       "std_train_score                                             0.0110156   \n",
       "\n",
       "                                                                   15  \n",
       "mean_fit_time                                                  15.401  \n",
       "mean_score_time                                                0.0614  \n",
       "mean_test_score                                             -0.814235  \n",
       "mean_train_score                                            -0.787111  \n",
       "param_max_depth                                                     3  \n",
       "param_max_features                                                0.9  \n",
       "param_n_estimators                                                  7  \n",
       "param_oob_score                                                  True  \n",
       "params              {'max_depth': 3, 'max_features': 0.9, 'n_estim...  \n",
       "rank_test_score                                                   100  \n",
       "split0_test_score                                           -0.775777  \n",
       "split0_train_score                                          -0.818307  \n",
       "split1_test_score                                           -0.706675  \n",
       "split1_train_score                                          -0.796535  \n",
       "split2_test_score                                            -0.76362  \n",
       "split2_train_score                                           -0.76615  \n",
       "split3_test_score                                           -0.887351  \n",
       "split3_train_score                                          -0.765027  \n",
       "split4_test_score                                           -0.932289  \n",
       "split4_train_score                                          -0.789538  \n",
       "std_fit_time                                                  7.81093  \n",
       "std_score_time                                              0.0159824  \n",
       "std_test_score                                              0.0844375  \n",
       "std_train_score                                             0.0199749  \n",
       "\n",
       "[24 rows x 100 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_result=pd.DataFrame(clf.cv_results_).sort_values('rank_test_score')\n",
    "cv_result.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 4, 'max_features': 0.5, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.7, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.7, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.5, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.7, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.7, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.5, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.7, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.7, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.5, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.5, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.5, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.7, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.5, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.7, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.5, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.7, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.5, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.7, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.5, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.5, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.7, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.5, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.7, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.2, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.2, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.2, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.2, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.2, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.5, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.5, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.7, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.5, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.7, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.5, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.5, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.7, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.7, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.7, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.5, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.7, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 4, 'max_features': 0.2, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.2, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.2, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.2, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.2, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.2, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.2, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.2, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.2, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.2, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.2, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 2, 'max_features': 0.2, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 3, 'max_features': 0.2, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.5, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.5, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.5, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.5, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.7, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.5, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.7, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.7, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.7, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.5, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.7, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.7, 'n_estimators': 5, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.2, 'n_estimators': 10, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.2, 'n_estimators': 13, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.2, 'n_estimators': 15, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.2, 'n_estimators': 20, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.2, 'n_estimators': 7, 'oob_score': True}\n",
      "{'max_depth': 1, 'max_features': 0.2, 'n_estimators': 5, 'oob_score': True}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    }
   ],
   "source": [
    "# for i in pd.DataFrame(clf.cv_results_).sort_values('rank_test_score').params:\n",
    "#     print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>65</th>\n",
       "      <th>70</th>\n",
       "      <th>71</th>\n",
       "      <th>64</th>\n",
       "      <th>69</th>\n",
       "      <th>67</th>\n",
       "      <th>63</th>\n",
       "      <th>68</th>\n",
       "      <th>66</th>\n",
       "      <th>62</th>\n",
       "      <th>...</th>\n",
       "      <th>14</th>\n",
       "      <th>6</th>\n",
       "      <th>15</th>\n",
       "      <th>12</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean_fit_time</th>\n",
       "      <td>35.2432</td>\n",
       "      <td>28.6572</td>\n",
       "      <td>31.9064</td>\n",
       "      <td>25.9338</td>\n",
       "      <td>25.9946</td>\n",
       "      <td>14.0286</td>\n",
       "      <td>22.9574</td>\n",
       "      <td>19.8292</td>\n",
       "      <td>10.0284</td>\n",
       "      <td>17.7614</td>\n",
       "      <td>...</td>\n",
       "      <td>6.509</td>\n",
       "      <td>3.322</td>\n",
       "      <td>7.9986</td>\n",
       "      <td>3.752</td>\n",
       "      <td>2.6018</td>\n",
       "      <td>3.8334</td>\n",
       "      <td>3.9782</td>\n",
       "      <td>5.5878</td>\n",
       "      <td>1.6542</td>\n",
       "      <td>0.755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_score_time</th>\n",
       "      <td>0.169</td>\n",
       "      <td>0.1104</td>\n",
       "      <td>0.1282</td>\n",
       "      <td>0.1272</td>\n",
       "      <td>0.105</td>\n",
       "      <td>0.0654</td>\n",
       "      <td>0.1088</td>\n",
       "      <td>0.0931999</td>\n",
       "      <td>0.0582</td>\n",
       "      <td>0.0863999</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0646</td>\n",
       "      <td>0.0429999</td>\n",
       "      <td>0.0776</td>\n",
       "      <td>0.0374</td>\n",
       "      <td>0.0798001</td>\n",
       "      <td>0.0844</td>\n",
       "      <td>0.1086</td>\n",
       "      <td>0.1208</td>\n",
       "      <td>0.0466001</td>\n",
       "      <td>0.0221999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_test_score</th>\n",
       "      <td>-0.808654</td>\n",
       "      <td>-0.808688</td>\n",
       "      <td>-0.808793</td>\n",
       "      <td>-0.808799</td>\n",
       "      <td>-0.809102</td>\n",
       "      <td>-0.809505</td>\n",
       "      <td>-0.809523</td>\n",
       "      <td>-0.809529</td>\n",
       "      <td>-0.809616</td>\n",
       "      <td>-0.810487</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.923638</td>\n",
       "      <td>-0.92366</td>\n",
       "      <td>-0.924244</td>\n",
       "      <td>-0.924265</td>\n",
       "      <td>-0.931954</td>\n",
       "      <td>-0.931994</td>\n",
       "      <td>-0.936331</td>\n",
       "      <td>-0.937916</td>\n",
       "      <td>-0.940094</td>\n",
       "      <td>-0.952866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_train_score</th>\n",
       "      <td>-0.777401</td>\n",
       "      <td>-0.777813</td>\n",
       "      <td>-0.777525</td>\n",
       "      <td>-0.777442</td>\n",
       "      <td>-0.777924</td>\n",
       "      <td>-0.778238</td>\n",
       "      <td>-0.77763</td>\n",
       "      <td>-0.777819</td>\n",
       "      <td>-0.778646</td>\n",
       "      <td>-0.77772</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.902799</td>\n",
       "      <td>-0.903175</td>\n",
       "      <td>-0.903436</td>\n",
       "      <td>-0.903418</td>\n",
       "      <td>-0.912226</td>\n",
       "      <td>-0.911815</td>\n",
       "      <td>-0.915699</td>\n",
       "      <td>-0.91598</td>\n",
       "      <td>-0.919761</td>\n",
       "      <td>-0.930078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_max_depth</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_max_features</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_n_estimators</th>\n",
       "      <td>20</td>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "      <td>15</td>\n",
       "      <td>13</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>param_oob_score</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>params</th>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.5, 'n_estim...</td>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.5, 'n_estim...</td>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.5, 'n_estim...</td>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 4, 'max_features': 0.5, 'n_estim...</td>\n",
       "      <td>...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.5, 'n_estim...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.7, 'n_estim...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.2, 'n_estim...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.2, 'n_estim...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.2, 'n_estim...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.2, 'n_estim...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.2, 'n_estim...</td>\n",
       "      <td>{'max_depth': 1, 'max_features': 0.2, 'n_estim...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rank_test_score</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>63</td>\n",
       "      <td>64</td>\n",
       "      <td>65</td>\n",
       "      <td>66</td>\n",
       "      <td>67</td>\n",
       "      <td>68</td>\n",
       "      <td>69</td>\n",
       "      <td>70</td>\n",
       "      <td>71</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split0_test_score</th>\n",
       "      <td>-0.776417</td>\n",
       "      <td>-0.776371</td>\n",
       "      <td>-0.776721</td>\n",
       "      <td>-0.776074</td>\n",
       "      <td>-0.777668</td>\n",
       "      <td>-0.77835</td>\n",
       "      <td>-0.777898</td>\n",
       "      <td>-0.778204</td>\n",
       "      <td>-0.778815</td>\n",
       "      <td>-0.779065</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.901423</td>\n",
       "      <td>-0.901134</td>\n",
       "      <td>-0.902428</td>\n",
       "      <td>-0.904171</td>\n",
       "      <td>-0.905945</td>\n",
       "      <td>-0.909013</td>\n",
       "      <td>-0.913</td>\n",
       "      <td>-0.91788</td>\n",
       "      <td>-0.913859</td>\n",
       "      <td>-0.929254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split0_train_score</th>\n",
       "      <td>-0.805383</td>\n",
       "      <td>-0.806267</td>\n",
       "      <td>-0.805351</td>\n",
       "      <td>-0.805103</td>\n",
       "      <td>-0.806142</td>\n",
       "      <td>-0.807838</td>\n",
       "      <td>-0.805163</td>\n",
       "      <td>-0.806022</td>\n",
       "      <td>-0.809353</td>\n",
       "      <td>-0.805798</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.922697</td>\n",
       "      <td>-0.924646</td>\n",
       "      <td>-0.923552</td>\n",
       "      <td>-0.925858</td>\n",
       "      <td>-0.933932</td>\n",
       "      <td>-0.931511</td>\n",
       "      <td>-0.933701</td>\n",
       "      <td>-0.931858</td>\n",
       "      <td>-0.940699</td>\n",
       "      <td>-0.948487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split1_test_score</th>\n",
       "      <td>-0.70098</td>\n",
       "      <td>-0.699825</td>\n",
       "      <td>-0.699974</td>\n",
       "      <td>-0.70091</td>\n",
       "      <td>-0.700439</td>\n",
       "      <td>-0.701553</td>\n",
       "      <td>-0.702118</td>\n",
       "      <td>-0.701103</td>\n",
       "      <td>-0.701122</td>\n",
       "      <td>-0.703869</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.830549</td>\n",
       "      <td>-0.828798</td>\n",
       "      <td>-0.827701</td>\n",
       "      <td>-0.828798</td>\n",
       "      <td>-0.84242</td>\n",
       "      <td>-0.842461</td>\n",
       "      <td>-0.846887</td>\n",
       "      <td>-0.8492</td>\n",
       "      <td>-0.850441</td>\n",
       "      <td>-0.862509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split1_train_score</th>\n",
       "      <td>-0.786731</td>\n",
       "      <td>-0.787445</td>\n",
       "      <td>-0.787439</td>\n",
       "      <td>-0.786691</td>\n",
       "      <td>-0.787648</td>\n",
       "      <td>-0.787469</td>\n",
       "      <td>-0.786896</td>\n",
       "      <td>-0.787699</td>\n",
       "      <td>-0.787516</td>\n",
       "      <td>-0.786955</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.905565</td>\n",
       "      <td>-0.904452</td>\n",
       "      <td>-0.903753</td>\n",
       "      <td>-0.904452</td>\n",
       "      <td>-0.918931</td>\n",
       "      <td>-0.918118</td>\n",
       "      <td>-0.922544</td>\n",
       "      <td>-0.923381</td>\n",
       "      <td>-0.926204</td>\n",
       "      <td>-0.935934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split2_test_score</th>\n",
       "      <td>-0.759619</td>\n",
       "      <td>-0.758711</td>\n",
       "      <td>-0.758567</td>\n",
       "      <td>-0.759518</td>\n",
       "      <td>-0.758649</td>\n",
       "      <td>-0.758659</td>\n",
       "      <td>-0.759538</td>\n",
       "      <td>-0.758644</td>\n",
       "      <td>-0.759228</td>\n",
       "      <td>-0.759664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.890298</td>\n",
       "      <td>-0.887876</td>\n",
       "      <td>-0.892494</td>\n",
       "      <td>-0.887876</td>\n",
       "      <td>-0.91282</td>\n",
       "      <td>-0.912015</td>\n",
       "      <td>-0.918205</td>\n",
       "      <td>-0.919145</td>\n",
       "      <td>-0.9226</td>\n",
       "      <td>-0.93517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split2_train_score</th>\n",
       "      <td>-0.757537</td>\n",
       "      <td>-0.757836</td>\n",
       "      <td>-0.757685</td>\n",
       "      <td>-0.757634</td>\n",
       "      <td>-0.757968</td>\n",
       "      <td>-0.757949</td>\n",
       "      <td>-0.757935</td>\n",
       "      <td>-0.757652</td>\n",
       "      <td>-0.758065</td>\n",
       "      <td>-0.757882</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.883222</td>\n",
       "      <td>-0.881092</td>\n",
       "      <td>-0.885431</td>\n",
       "      <td>-0.881092</td>\n",
       "      <td>-0.89183</td>\n",
       "      <td>-0.89277</td>\n",
       "      <td>-0.897169</td>\n",
       "      <td>-0.898729</td>\n",
       "      <td>-0.899535</td>\n",
       "      <td>-0.910659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split3_test_score</th>\n",
       "      <td>-0.874799</td>\n",
       "      <td>-0.87677</td>\n",
       "      <td>-0.87681</td>\n",
       "      <td>-0.875246</td>\n",
       "      <td>-0.877166</td>\n",
       "      <td>-0.87673</td>\n",
       "      <td>-0.875596</td>\n",
       "      <td>-0.877073</td>\n",
       "      <td>-0.875753</td>\n",
       "      <td>-0.877056</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.983296</td>\n",
       "      <td>-0.9859</td>\n",
       "      <td>-0.984267</td>\n",
       "      <td>-0.9859</td>\n",
       "      <td>-0.993049</td>\n",
       "      <td>-0.992011</td>\n",
       "      <td>-0.9962</td>\n",
       "      <td>-0.996012</td>\n",
       "      <td>-1.00367</td>\n",
       "      <td>-1.01978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split3_train_score</th>\n",
       "      <td>-0.757488</td>\n",
       "      <td>-0.757375</td>\n",
       "      <td>-0.757241</td>\n",
       "      <td>-0.757639</td>\n",
       "      <td>-0.757563</td>\n",
       "      <td>-0.757664</td>\n",
       "      <td>-0.757924</td>\n",
       "      <td>-0.7575</td>\n",
       "      <td>-0.757676</td>\n",
       "      <td>-0.757936</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.887705</td>\n",
       "      <td>-0.889839</td>\n",
       "      <td>-0.888525</td>\n",
       "      <td>-0.889839</td>\n",
       "      <td>-0.897589</td>\n",
       "      <td>-0.897622</td>\n",
       "      <td>-0.901798</td>\n",
       "      <td>-0.902639</td>\n",
       "      <td>-0.905167</td>\n",
       "      <td>-0.917926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split4_test_score</th>\n",
       "      <td>-0.925932</td>\n",
       "      <td>-0.926254</td>\n",
       "      <td>-0.92637</td>\n",
       "      <td>-0.926687</td>\n",
       "      <td>-0.926093</td>\n",
       "      <td>-0.926669</td>\n",
       "      <td>-0.926889</td>\n",
       "      <td>-0.927056</td>\n",
       "      <td>-0.927559</td>\n",
       "      <td>-0.927203</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.00928</td>\n",
       "      <td>-1.01113</td>\n",
       "      <td>-1.01106</td>\n",
       "      <td>-1.01113</td>\n",
       "      <td>-1.00364</td>\n",
       "      <td>-1.00257</td>\n",
       "      <td>-1.00562</td>\n",
       "      <td>-1.00562</td>\n",
       "      <td>-1.00835</td>\n",
       "      <td>-1.01645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split4_train_score</th>\n",
       "      <td>-0.779867</td>\n",
       "      <td>-0.780144</td>\n",
       "      <td>-0.779908</td>\n",
       "      <td>-0.780143</td>\n",
       "      <td>-0.780299</td>\n",
       "      <td>-0.78027</td>\n",
       "      <td>-0.78023</td>\n",
       "      <td>-0.780222</td>\n",
       "      <td>-0.780617</td>\n",
       "      <td>-0.780029</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.914802</td>\n",
       "      <td>-0.915848</td>\n",
       "      <td>-0.915916</td>\n",
       "      <td>-0.915848</td>\n",
       "      <td>-0.918845</td>\n",
       "      <td>-0.919055</td>\n",
       "      <td>-0.923281</td>\n",
       "      <td>-0.923293</td>\n",
       "      <td>-0.927202</td>\n",
       "      <td>-0.937385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_fit_time</th>\n",
       "      <td>18.4225</td>\n",
       "      <td>13.4028</td>\n",
       "      <td>12.7443</td>\n",
       "      <td>12.6402</td>\n",
       "      <td>12.7392</td>\n",
       "      <td>6.98992</td>\n",
       "      <td>11.7056</td>\n",
       "      <td>9.88117</td>\n",
       "      <td>4.95177</td>\n",
       "      <td>8.78384</td>\n",
       "      <td>...</td>\n",
       "      <td>3.12054</td>\n",
       "      <td>1.62578</td>\n",
       "      <td>3.79954</td>\n",
       "      <td>2.05845</td>\n",
       "      <td>1.49709</td>\n",
       "      <td>1.8628</td>\n",
       "      <td>2.0367</td>\n",
       "      <td>2.75744</td>\n",
       "      <td>1.15738</td>\n",
       "      <td>0.352707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_score_time</th>\n",
       "      <td>0.0161246</td>\n",
       "      <td>0.0115689</td>\n",
       "      <td>0.0196205</td>\n",
       "      <td>0.00921733</td>\n",
       "      <td>0.00352149</td>\n",
       "      <td>0.00605316</td>\n",
       "      <td>0.00466471</td>\n",
       "      <td>0.0147702</td>\n",
       "      <td>0.0148917</td>\n",
       "      <td>0.00527631</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00776129</td>\n",
       "      <td>0.00933813</td>\n",
       "      <td>0.0127216</td>\n",
       "      <td>0.00279996</td>\n",
       "      <td>0.0202129</td>\n",
       "      <td>0.019602</td>\n",
       "      <td>0.0130323</td>\n",
       "      <td>0.0230079</td>\n",
       "      <td>0.0230182</td>\n",
       "      <td>0.00487454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_test_score</th>\n",
       "      <td>0.0823986</td>\n",
       "      <td>0.0832206</td>\n",
       "      <td>0.0832113</td>\n",
       "      <td>0.0827531</td>\n",
       "      <td>0.0829795</td>\n",
       "      <td>0.0827337</td>\n",
       "      <td>0.0824039</td>\n",
       "      <td>0.0830366</td>\n",
       "      <td>0.082862</td>\n",
       "      <td>0.0821602</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0657247</td>\n",
       "      <td>0.0674591</td>\n",
       "      <td>0.0669427</td>\n",
       "      <td>0.0672675</td>\n",
       "      <td>0.0604193</td>\n",
       "      <td>0.0597375</td>\n",
       "      <td>0.0592745</td>\n",
       "      <td>0.0581053</td>\n",
       "      <td>0.0600359</td>\n",
       "      <td>0.0596612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_train_score</th>\n",
       "      <td>0.0182606</td>\n",
       "      <td>0.0185722</td>\n",
       "      <td>0.0183484</td>\n",
       "      <td>0.0181244</td>\n",
       "      <td>0.0184892</td>\n",
       "      <td>0.0189759</td>\n",
       "      <td>0.0180382</td>\n",
       "      <td>0.0185384</td>\n",
       "      <td>0.0194361</td>\n",
       "      <td>0.0182426</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0152233</td>\n",
       "      <td>0.0160546</td>\n",
       "      <td>0.0148794</td>\n",
       "      <td>0.0163828</td>\n",
       "      <td>0.015428</td>\n",
       "      <td>0.0144516</td>\n",
       "      <td>0.0138926</td>\n",
       "      <td>0.0129301</td>\n",
       "      <td>0.0152141</td>\n",
       "      <td>0.0137941</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24 rows × 72 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                   65  \\\n",
       "mean_fit_time                                                 35.2432   \n",
       "mean_score_time                                                 0.169   \n",
       "mean_test_score                                             -0.808654   \n",
       "mean_train_score                                            -0.777401   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.5   \n",
       "param_n_estimators                                                 20   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.5, 'n_estim...   \n",
       "rank_test_score                                                     1   \n",
       "split0_test_score                                           -0.776417   \n",
       "split0_train_score                                          -0.805383   \n",
       "split1_test_score                                            -0.70098   \n",
       "split1_train_score                                          -0.786731   \n",
       "split2_test_score                                           -0.759619   \n",
       "split2_train_score                                          -0.757537   \n",
       "split3_test_score                                           -0.874799   \n",
       "split3_train_score                                          -0.757488   \n",
       "split4_test_score                                           -0.925932   \n",
       "split4_train_score                                          -0.779867   \n",
       "std_fit_time                                                  18.4225   \n",
       "std_score_time                                              0.0161246   \n",
       "std_test_score                                              0.0823986   \n",
       "std_train_score                                             0.0182606   \n",
       "\n",
       "                                                                   70  \\\n",
       "mean_fit_time                                                 28.6572   \n",
       "mean_score_time                                                0.1104   \n",
       "mean_test_score                                             -0.808688   \n",
       "mean_train_score                                            -0.777813   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                 15   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                     2   \n",
       "split0_test_score                                           -0.776371   \n",
       "split0_train_score                                          -0.806267   \n",
       "split1_test_score                                           -0.699825   \n",
       "split1_train_score                                          -0.787445   \n",
       "split2_test_score                                           -0.758711   \n",
       "split2_train_score                                          -0.757836   \n",
       "split3_test_score                                            -0.87677   \n",
       "split3_train_score                                          -0.757375   \n",
       "split4_test_score                                           -0.926254   \n",
       "split4_train_score                                          -0.780144   \n",
       "std_fit_time                                                  13.4028   \n",
       "std_score_time                                              0.0115689   \n",
       "std_test_score                                              0.0832206   \n",
       "std_train_score                                             0.0185722   \n",
       "\n",
       "                                                                   71  \\\n",
       "mean_fit_time                                                 31.9064   \n",
       "mean_score_time                                                0.1282   \n",
       "mean_test_score                                             -0.808793   \n",
       "mean_train_score                                            -0.777525   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                 20   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                     3   \n",
       "split0_test_score                                           -0.776721   \n",
       "split0_train_score                                          -0.805351   \n",
       "split1_test_score                                           -0.699974   \n",
       "split1_train_score                                          -0.787439   \n",
       "split2_test_score                                           -0.758567   \n",
       "split2_train_score                                          -0.757685   \n",
       "split3_test_score                                            -0.87681   \n",
       "split3_train_score                                          -0.757241   \n",
       "split4_test_score                                            -0.92637   \n",
       "split4_train_score                                          -0.779908   \n",
       "std_fit_time                                                  12.7443   \n",
       "std_score_time                                              0.0196205   \n",
       "std_test_score                                              0.0832113   \n",
       "std_train_score                                             0.0183484   \n",
       "\n",
       "                                                                   64  \\\n",
       "mean_fit_time                                                 25.9338   \n",
       "mean_score_time                                                0.1272   \n",
       "mean_test_score                                             -0.808799   \n",
       "mean_train_score                                            -0.777442   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.5   \n",
       "param_n_estimators                                                 15   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.5, 'n_estim...   \n",
       "rank_test_score                                                     4   \n",
       "split0_test_score                                           -0.776074   \n",
       "split0_train_score                                          -0.805103   \n",
       "split1_test_score                                            -0.70091   \n",
       "split1_train_score                                          -0.786691   \n",
       "split2_test_score                                           -0.759518   \n",
       "split2_train_score                                          -0.757634   \n",
       "split3_test_score                                           -0.875246   \n",
       "split3_train_score                                          -0.757639   \n",
       "split4_test_score                                           -0.926687   \n",
       "split4_train_score                                          -0.780143   \n",
       "std_fit_time                                                  12.6402   \n",
       "std_score_time                                             0.00921733   \n",
       "std_test_score                                              0.0827531   \n",
       "std_train_score                                             0.0181244   \n",
       "\n",
       "                                                                   69  \\\n",
       "mean_fit_time                                                 25.9946   \n",
       "mean_score_time                                                 0.105   \n",
       "mean_test_score                                             -0.809102   \n",
       "mean_train_score                                            -0.777924   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                 13   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                     5   \n",
       "split0_test_score                                           -0.777668   \n",
       "split0_train_score                                          -0.806142   \n",
       "split1_test_score                                           -0.700439   \n",
       "split1_train_score                                          -0.787648   \n",
       "split2_test_score                                           -0.758649   \n",
       "split2_train_score                                          -0.757968   \n",
       "split3_test_score                                           -0.877166   \n",
       "split3_train_score                                          -0.757563   \n",
       "split4_test_score                                           -0.926093   \n",
       "split4_train_score                                          -0.780299   \n",
       "std_fit_time                                                  12.7392   \n",
       "std_score_time                                             0.00352149   \n",
       "std_test_score                                              0.0829795   \n",
       "std_train_score                                             0.0184892   \n",
       "\n",
       "                                                                   67  \\\n",
       "mean_fit_time                                                 14.0286   \n",
       "mean_score_time                                                0.0654   \n",
       "mean_test_score                                             -0.809505   \n",
       "mean_train_score                                            -0.778238   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                  7   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                     6   \n",
       "split0_test_score                                            -0.77835   \n",
       "split0_train_score                                          -0.807838   \n",
       "split1_test_score                                           -0.701553   \n",
       "split1_train_score                                          -0.787469   \n",
       "split2_test_score                                           -0.758659   \n",
       "split2_train_score                                          -0.757949   \n",
       "split3_test_score                                            -0.87673   \n",
       "split3_train_score                                          -0.757664   \n",
       "split4_test_score                                           -0.926669   \n",
       "split4_train_score                                           -0.78027   \n",
       "std_fit_time                                                  6.98992   \n",
       "std_score_time                                             0.00605316   \n",
       "std_test_score                                              0.0827337   \n",
       "std_train_score                                             0.0189759   \n",
       "\n",
       "                                                                   63  \\\n",
       "mean_fit_time                                                 22.9574   \n",
       "mean_score_time                                                0.1088   \n",
       "mean_test_score                                             -0.809523   \n",
       "mean_train_score                                             -0.77763   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.5   \n",
       "param_n_estimators                                                 13   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.5, 'n_estim...   \n",
       "rank_test_score                                                     7   \n",
       "split0_test_score                                           -0.777898   \n",
       "split0_train_score                                          -0.805163   \n",
       "split1_test_score                                           -0.702118   \n",
       "split1_train_score                                          -0.786896   \n",
       "split2_test_score                                           -0.759538   \n",
       "split2_train_score                                          -0.757935   \n",
       "split3_test_score                                           -0.875596   \n",
       "split3_train_score                                          -0.757924   \n",
       "split4_test_score                                           -0.926889   \n",
       "split4_train_score                                           -0.78023   \n",
       "std_fit_time                                                  11.7056   \n",
       "std_score_time                                             0.00466471   \n",
       "std_test_score                                              0.0824039   \n",
       "std_train_score                                             0.0180382   \n",
       "\n",
       "                                                                   68  \\\n",
       "mean_fit_time                                                 19.8292   \n",
       "mean_score_time                                             0.0931999   \n",
       "mean_test_score                                             -0.809529   \n",
       "mean_train_score                                            -0.777819   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                 10   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                     8   \n",
       "split0_test_score                                           -0.778204   \n",
       "split0_train_score                                          -0.806022   \n",
       "split1_test_score                                           -0.701103   \n",
       "split1_train_score                                          -0.787699   \n",
       "split2_test_score                                           -0.758644   \n",
       "split2_train_score                                          -0.757652   \n",
       "split3_test_score                                           -0.877073   \n",
       "split3_train_score                                            -0.7575   \n",
       "split4_test_score                                           -0.927056   \n",
       "split4_train_score                                          -0.780222   \n",
       "std_fit_time                                                  9.88117   \n",
       "std_score_time                                              0.0147702   \n",
       "std_test_score                                              0.0830366   \n",
       "std_train_score                                             0.0185384   \n",
       "\n",
       "                                                                   66  \\\n",
       "mean_fit_time                                                 10.0284   \n",
       "mean_score_time                                                0.0582   \n",
       "mean_test_score                                             -0.809616   \n",
       "mean_train_score                                            -0.778646   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                  5   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                     9   \n",
       "split0_test_score                                           -0.778815   \n",
       "split0_train_score                                          -0.809353   \n",
       "split1_test_score                                           -0.701122   \n",
       "split1_train_score                                          -0.787516   \n",
       "split2_test_score                                           -0.759228   \n",
       "split2_train_score                                          -0.758065   \n",
       "split3_test_score                                           -0.875753   \n",
       "split3_train_score                                          -0.757676   \n",
       "split4_test_score                                           -0.927559   \n",
       "split4_train_score                                          -0.780617   \n",
       "std_fit_time                                                  4.95177   \n",
       "std_score_time                                              0.0148917   \n",
       "std_test_score                                               0.082862   \n",
       "std_train_score                                             0.0194361   \n",
       "\n",
       "                                                                   62  \\\n",
       "mean_fit_time                                                 17.7614   \n",
       "mean_score_time                                             0.0863999   \n",
       "mean_test_score                                             -0.810487   \n",
       "mean_train_score                                             -0.77772   \n",
       "param_max_depth                                                     4   \n",
       "param_max_features                                                0.5   \n",
       "param_n_estimators                                                 10   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 4, 'max_features': 0.5, 'n_estim...   \n",
       "rank_test_score                                                    10   \n",
       "split0_test_score                                           -0.779065   \n",
       "split0_train_score                                          -0.805798   \n",
       "split1_test_score                                           -0.703869   \n",
       "split1_train_score                                          -0.786955   \n",
       "split2_test_score                                           -0.759664   \n",
       "split2_train_score                                          -0.757882   \n",
       "split3_test_score                                           -0.877056   \n",
       "split3_train_score                                          -0.757936   \n",
       "split4_test_score                                           -0.927203   \n",
       "split4_train_score                                          -0.780029   \n",
       "std_fit_time                                                  8.78384   \n",
       "std_score_time                                             0.00527631   \n",
       "std_test_score                                              0.0821602   \n",
       "std_train_score                                             0.0182426   \n",
       "\n",
       "                                          ...                          \\\n",
       "mean_fit_time                             ...                           \n",
       "mean_score_time                           ...                           \n",
       "mean_test_score                           ...                           \n",
       "mean_train_score                          ...                           \n",
       "param_max_depth                           ...                           \n",
       "param_max_features                        ...                           \n",
       "param_n_estimators                        ...                           \n",
       "param_oob_score                           ...                           \n",
       "params                                    ...                           \n",
       "rank_test_score                           ...                           \n",
       "split0_test_score                         ...                           \n",
       "split0_train_score                        ...                           \n",
       "split1_test_score                         ...                           \n",
       "split1_train_score                        ...                           \n",
       "split2_test_score                         ...                           \n",
       "split2_train_score                        ...                           \n",
       "split3_test_score                         ...                           \n",
       "split3_train_score                        ...                           \n",
       "split4_test_score                         ...                           \n",
       "split4_train_score                        ...                           \n",
       "std_fit_time                              ...                           \n",
       "std_score_time                            ...                           \n",
       "std_test_score                            ...                           \n",
       "std_train_score                           ...                           \n",
       "\n",
       "                                                                   14  \\\n",
       "mean_fit_time                                                   6.509   \n",
       "mean_score_time                                                0.0646   \n",
       "mean_test_score                                             -0.923638   \n",
       "mean_train_score                                            -0.902799   \n",
       "param_max_depth                                                     1   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                 10   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 1, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                    63   \n",
       "split0_test_score                                           -0.901423   \n",
       "split0_train_score                                          -0.922697   \n",
       "split1_test_score                                           -0.830549   \n",
       "split1_train_score                                          -0.905565   \n",
       "split2_test_score                                           -0.890298   \n",
       "split2_train_score                                          -0.883222   \n",
       "split3_test_score                                           -0.983296   \n",
       "split3_train_score                                          -0.887705   \n",
       "split4_test_score                                            -1.00928   \n",
       "split4_train_score                                          -0.914802   \n",
       "std_fit_time                                                  3.12054   \n",
       "std_score_time                                             0.00776129   \n",
       "std_test_score                                              0.0657247   \n",
       "std_train_score                                             0.0152233   \n",
       "\n",
       "                                                                   6   \\\n",
       "mean_fit_time                                                   3.322   \n",
       "mean_score_time                                             0.0429999   \n",
       "mean_test_score                                              -0.92366   \n",
       "mean_train_score                                            -0.903175   \n",
       "param_max_depth                                                     1   \n",
       "param_max_features                                                0.5   \n",
       "param_n_estimators                                                  5   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 1, 'max_features': 0.5, 'n_estim...   \n",
       "rank_test_score                                                    64   \n",
       "split0_test_score                                           -0.901134   \n",
       "split0_train_score                                          -0.924646   \n",
       "split1_test_score                                           -0.828798   \n",
       "split1_train_score                                          -0.904452   \n",
       "split2_test_score                                           -0.887876   \n",
       "split2_train_score                                          -0.881092   \n",
       "split3_test_score                                             -0.9859   \n",
       "split3_train_score                                          -0.889839   \n",
       "split4_test_score                                            -1.01113   \n",
       "split4_train_score                                          -0.915848   \n",
       "std_fit_time                                                  1.62578   \n",
       "std_score_time                                             0.00933813   \n",
       "std_test_score                                              0.0674591   \n",
       "std_train_score                                             0.0160546   \n",
       "\n",
       "                                                                   15  \\\n",
       "mean_fit_time                                                  7.9986   \n",
       "mean_score_time                                                0.0776   \n",
       "mean_test_score                                             -0.924244   \n",
       "mean_train_score                                            -0.903436   \n",
       "param_max_depth                                                     1   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                 13   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 1, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                    65   \n",
       "split0_test_score                                           -0.902428   \n",
       "split0_train_score                                          -0.923552   \n",
       "split1_test_score                                           -0.827701   \n",
       "split1_train_score                                          -0.903753   \n",
       "split2_test_score                                           -0.892494   \n",
       "split2_train_score                                          -0.885431   \n",
       "split3_test_score                                           -0.984267   \n",
       "split3_train_score                                          -0.888525   \n",
       "split4_test_score                                            -1.01106   \n",
       "split4_train_score                                          -0.915916   \n",
       "std_fit_time                                                  3.79954   \n",
       "std_score_time                                              0.0127216   \n",
       "std_test_score                                              0.0669427   \n",
       "std_train_score                                             0.0148794   \n",
       "\n",
       "                                                                   12  \\\n",
       "mean_fit_time                                                   3.752   \n",
       "mean_score_time                                                0.0374   \n",
       "mean_test_score                                             -0.924265   \n",
       "mean_train_score                                            -0.903418   \n",
       "param_max_depth                                                     1   \n",
       "param_max_features                                                0.7   \n",
       "param_n_estimators                                                  5   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 1, 'max_features': 0.7, 'n_estim...   \n",
       "rank_test_score                                                    66   \n",
       "split0_test_score                                           -0.904171   \n",
       "split0_train_score                                          -0.925858   \n",
       "split1_test_score                                           -0.828798   \n",
       "split1_train_score                                          -0.904452   \n",
       "split2_test_score                                           -0.887876   \n",
       "split2_train_score                                          -0.881092   \n",
       "split3_test_score                                             -0.9859   \n",
       "split3_train_score                                          -0.889839   \n",
       "split4_test_score                                            -1.01113   \n",
       "split4_train_score                                          -0.915848   \n",
       "std_fit_time                                                  2.05845   \n",
       "std_score_time                                             0.00279996   \n",
       "std_test_score                                              0.0672675   \n",
       "std_train_score                                             0.0163828   \n",
       "\n",
       "                                                                   2   \\\n",
       "mean_fit_time                                                  2.6018   \n",
       "mean_score_time                                             0.0798001   \n",
       "mean_test_score                                             -0.931954   \n",
       "mean_train_score                                            -0.912226   \n",
       "param_max_depth                                                     1   \n",
       "param_max_features                                                0.2   \n",
       "param_n_estimators                                                 10   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 1, 'max_features': 0.2, 'n_estim...   \n",
       "rank_test_score                                                    67   \n",
       "split0_test_score                                           -0.905945   \n",
       "split0_train_score                                          -0.933932   \n",
       "split1_test_score                                            -0.84242   \n",
       "split1_train_score                                          -0.918931   \n",
       "split2_test_score                                            -0.91282   \n",
       "split2_train_score                                           -0.89183   \n",
       "split3_test_score                                           -0.993049   \n",
       "split3_train_score                                          -0.897589   \n",
       "split4_test_score                                            -1.00364   \n",
       "split4_train_score                                          -0.918845   \n",
       "std_fit_time                                                  1.49709   \n",
       "std_score_time                                              0.0202129   \n",
       "std_test_score                                              0.0604193   \n",
       "std_train_score                                              0.015428   \n",
       "\n",
       "                                                                   3   \\\n",
       "mean_fit_time                                                  3.8334   \n",
       "mean_score_time                                                0.0844   \n",
       "mean_test_score                                             -0.931994   \n",
       "mean_train_score                                            -0.911815   \n",
       "param_max_depth                                                     1   \n",
       "param_max_features                                                0.2   \n",
       "param_n_estimators                                                 13   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 1, 'max_features': 0.2, 'n_estim...   \n",
       "rank_test_score                                                    68   \n",
       "split0_test_score                                           -0.909013   \n",
       "split0_train_score                                          -0.931511   \n",
       "split1_test_score                                           -0.842461   \n",
       "split1_train_score                                          -0.918118   \n",
       "split2_test_score                                           -0.912015   \n",
       "split2_train_score                                           -0.89277   \n",
       "split3_test_score                                           -0.992011   \n",
       "split3_train_score                                          -0.897622   \n",
       "split4_test_score                                            -1.00257   \n",
       "split4_train_score                                          -0.919055   \n",
       "std_fit_time                                                   1.8628   \n",
       "std_score_time                                               0.019602   \n",
       "std_test_score                                              0.0597375   \n",
       "std_train_score                                             0.0144516   \n",
       "\n",
       "                                                                   4   \\\n",
       "mean_fit_time                                                  3.9782   \n",
       "mean_score_time                                                0.1086   \n",
       "mean_test_score                                             -0.936331   \n",
       "mean_train_score                                            -0.915699   \n",
       "param_max_depth                                                     1   \n",
       "param_max_features                                                0.2   \n",
       "param_n_estimators                                                 15   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 1, 'max_features': 0.2, 'n_estim...   \n",
       "rank_test_score                                                    69   \n",
       "split0_test_score                                              -0.913   \n",
       "split0_train_score                                          -0.933701   \n",
       "split1_test_score                                           -0.846887   \n",
       "split1_train_score                                          -0.922544   \n",
       "split2_test_score                                           -0.918205   \n",
       "split2_train_score                                          -0.897169   \n",
       "split3_test_score                                             -0.9962   \n",
       "split3_train_score                                          -0.901798   \n",
       "split4_test_score                                            -1.00562   \n",
       "split4_train_score                                          -0.923281   \n",
       "std_fit_time                                                   2.0367   \n",
       "std_score_time                                              0.0130323   \n",
       "std_test_score                                              0.0592745   \n",
       "std_train_score                                             0.0138926   \n",
       "\n",
       "                                                                   5   \\\n",
       "mean_fit_time                                                  5.5878   \n",
       "mean_score_time                                                0.1208   \n",
       "mean_test_score                                             -0.937916   \n",
       "mean_train_score                                             -0.91598   \n",
       "param_max_depth                                                     1   \n",
       "param_max_features                                                0.2   \n",
       "param_n_estimators                                                 20   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 1, 'max_features': 0.2, 'n_estim...   \n",
       "rank_test_score                                                    70   \n",
       "split0_test_score                                            -0.91788   \n",
       "split0_train_score                                          -0.931858   \n",
       "split1_test_score                                             -0.8492   \n",
       "split1_train_score                                          -0.923381   \n",
       "split2_test_score                                           -0.919145   \n",
       "split2_train_score                                          -0.898729   \n",
       "split3_test_score                                           -0.996012   \n",
       "split3_train_score                                          -0.902639   \n",
       "split4_test_score                                            -1.00562   \n",
       "split4_train_score                                          -0.923293   \n",
       "std_fit_time                                                  2.75744   \n",
       "std_score_time                                              0.0230079   \n",
       "std_test_score                                              0.0581053   \n",
       "std_train_score                                             0.0129301   \n",
       "\n",
       "                                                                   1   \\\n",
       "mean_fit_time                                                  1.6542   \n",
       "mean_score_time                                             0.0466001   \n",
       "mean_test_score                                             -0.940094   \n",
       "mean_train_score                                            -0.919761   \n",
       "param_max_depth                                                     1   \n",
       "param_max_features                                                0.2   \n",
       "param_n_estimators                                                  7   \n",
       "param_oob_score                                                  True   \n",
       "params              {'max_depth': 1, 'max_features': 0.2, 'n_estim...   \n",
       "rank_test_score                                                    71   \n",
       "split0_test_score                                           -0.913859   \n",
       "split0_train_score                                          -0.940699   \n",
       "split1_test_score                                           -0.850441   \n",
       "split1_train_score                                          -0.926204   \n",
       "split2_test_score                                             -0.9226   \n",
       "split2_train_score                                          -0.899535   \n",
       "split3_test_score                                            -1.00367   \n",
       "split3_train_score                                          -0.905167   \n",
       "split4_test_score                                            -1.00835   \n",
       "split4_train_score                                          -0.927202   \n",
       "std_fit_time                                                  1.15738   \n",
       "std_score_time                                              0.0230182   \n",
       "std_test_score                                              0.0600359   \n",
       "std_train_score                                             0.0152141   \n",
       "\n",
       "                                                                   0   \n",
       "mean_fit_time                                                   0.755  \n",
       "mean_score_time                                             0.0221999  \n",
       "mean_test_score                                             -0.952866  \n",
       "mean_train_score                                            -0.930078  \n",
       "param_max_depth                                                     1  \n",
       "param_max_features                                                0.2  \n",
       "param_n_estimators                                                  5  \n",
       "param_oob_score                                                  True  \n",
       "params              {'max_depth': 1, 'max_features': 0.2, 'n_estim...  \n",
       "rank_test_score                                                    72  \n",
       "split0_test_score                                           -0.929254  \n",
       "split0_train_score                                          -0.948487  \n",
       "split1_test_score                                           -0.862509  \n",
       "split1_train_score                                          -0.935934  \n",
       "split2_test_score                                            -0.93517  \n",
       "split2_train_score                                          -0.910659  \n",
       "split3_test_score                                            -1.01978  \n",
       "split3_train_score                                          -0.917926  \n",
       "split4_test_score                                            -1.01645  \n",
       "split4_train_score                                          -0.937385  \n",
       "std_fit_time                                                 0.352707  \n",
       "std_score_time                                             0.00487454  \n",
       "std_test_score                                              0.0596612  \n",
       "std_train_score                                             0.0137941  \n",
       "\n",
       "[24 rows x 72 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cv_result=pd.DataFrame(clf.cv_results_).sort_values('rank_test_score')\n",
    "# cv_result.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params ={'max_depth': 5, 'max_features': 0.4, 'n_estimators': 25, 'oob_score': True,'random_state':1402}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:724: UserWarning: Some inputs do not have OOB scores. This probably means too few trees were used to compute any reliable oob estimates.\n",
      "  warn(\"Some inputs do not have OOB scores. \"\n"
     ]
    }
   ],
   "source": [
    "X,y = get_X_y_ensembling(all_oof_df)\n",
    "\n",
    "rf= RandomForestRegressor(**params)\n",
    "\n",
    "rf.fit(X,y)\n",
    "\n",
    "test_pred =  rf.predict(test_df)\n",
    "\n",
    "pd.Series(test_pred).describe()\n",
    "\n",
    "get_submission(test_pred,'ensembling_rf'); # .915 LB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    214200.000000\n",
       "mean          0.303533\n",
       "std           0.785487\n",
       "min           0.026790\n",
       "25%           0.030020\n",
       "50%           0.113811\n",
       "75%           0.281145\n",
       "max          19.404374\n",
       "dtype: float64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(test_pred).describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}