1_data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial-sally",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:24.384156Z",
     "iopub.status.busy": "2021-11-25T16:52:24.383161Z",
     "iopub.status.idle": "2021-11-25T16:52:24.397156Z",
     "shell.execute_reply": "2021-11-25T16:52:24.396159Z"
    },
    "papermill": {
     "duration": 0.049962,
     "end_time": "2021-11-25T16:52:24.397156",
     "exception": false,
     "start_time": "2021-11-25T16:52:24.347194",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "naughty-clearing",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:24.468776Z",
     "iopub.status.busy": "2021-11-25T16:52:24.467776Z",
     "iopub.status.idle": "2021-11-25T16:52:24.952738Z",
     "shell.execute_reply": "2021-11-25T16:52:24.952738Z"
    },
    "papermill": {
     "duration": 0.527579,
     "end_time": "2021-11-25T16:52:24.953738",
     "exception": false,
     "start_time": "2021-11-25T16:52:24.426159",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os.path as path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "emotional-referral",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:25.015738Z",
     "iopub.status.busy": "2021-11-25T16:52:25.014738Z",
     "iopub.status.idle": "2021-11-25T16:52:29.240128Z",
     "shell.execute_reply": "2021-11-25T16:52:29.240128Z"
    },
    "papermill": {
     "duration": 4.257385,
     "end_time": "2021-11-25T16:52:29.240128",
     "exception": false,
     "start_time": "2021-11-25T16:52:24.982743",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(path.join('1_rawdata', 'application_train.csv'))\n",
    "df_test = pd.read_csv(path.join(\"1_rawdata\", \"application_test.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "positive-correlation",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:29.301129Z",
     "iopub.status.busy": "2021-11-25T16:52:29.300129Z",
     "iopub.status.idle": "2021-11-25T16:52:29.699741Z",
     "shell.execute_reply": "2021-11-25T16:52:29.700740Z"
    },
    "papermill": {
     "duration": 0.431609,
     "end_time": "2021-11-25T16:52:29.700740",
     "exception": false,
     "start_time": "2021-11-25T16:52:29.269131",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# concat before dummies\n",
    "df = pd.concat([df_train, df_test], axis=\"index\")\n",
    "\n",
    "# easy data preparation by dropping all nulls\n",
    "# df = df.dropna(axis='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "beautiful-highway",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:29.780740Z",
     "iopub.status.busy": "2021-11-25T16:52:29.780740Z",
     "iopub.status.idle": "2021-11-25T16:52:29.811744Z",
     "shell.execute_reply": "2021-11-25T16:52:29.811744Z"
    },
    "papermill": {
     "duration": 0.083006,
     "end_time": "2021-11-25T16:52:29.811744",
     "exception": false,
     "start_time": "2021-11-25T16:52:29.728738",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "0      100002     1.0         Cash loans           M            N   \n",
       "1      100003     0.0         Cash loans           F            N   \n",
       "2      100004     0.0    Revolving loans           M            Y   \n",
       "3      100006     0.0         Cash loans           F            N   \n",
       "4      100007     0.0         Cash loans           M            N   \n",
       "\n",
       "  FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "0               Y             0          202500.0    406597.5      24700.5   \n",
       "1               N             0          270000.0   1293502.5      35698.5   \n",
       "2               Y             0           67500.0    135000.0       6750.0   \n",
       "3               Y             0          135000.0    312682.5      29686.5   \n",
       "4               Y             0          121500.0    513000.0      21865.5   \n",
       "\n",
       "   ...  FLAG_DOCUMENT_18 FLAG_DOCUMENT_19 FLAG_DOCUMENT_20 FLAG_DOCUMENT_21  \\\n",
       "0  ...                 0                0                0                0   \n",
       "1  ...                 0                0                0                0   \n",
       "2  ...                 0                0                0                0   \n",
       "3  ...                 0                0                0                0   \n",
       "4  ...                 0                0                0                0   \n",
       "\n",
       "  AMT_REQ_CREDIT_BUREAU_HOUR AMT_REQ_CREDIT_BUREAU_DAY  \\\n",
       "0                        0.0                       0.0   \n",
       "1                        0.0                       0.0   \n",
       "2                        0.0                       0.0   \n",
       "3                        NaN                       NaN   \n",
       "4                        0.0                       0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_WEEK  AMT_REQ_CREDIT_BUREAU_MON  \\\n",
       "0                         0.0                        0.0   \n",
       "1                         0.0                        0.0   \n",
       "2                         0.0                        0.0   \n",
       "3                         NaN                        NaN   \n",
       "4                         0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_QRT  AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                        0.0                         1.0  \n",
       "1                        0.0                         0.0  \n",
       "2                        0.0                         0.0  \n",
       "3                        NaN                         NaN  \n",
       "4                        0.0                         0.0  \n",
       "\n",
       "[5 rows x 122 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "moved-spouse",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:29.874742Z",
     "iopub.status.busy": "2021-11-25T16:52:29.873743Z",
     "iopub.status.idle": "2021-11-25T16:52:29.891739Z",
     "shell.execute_reply": "2021-11-25T16:52:29.890740Z"
    },
    "papermill": {
     "duration": 0.05,
     "end_time": "2021-11-25T16:52:29.891739",
     "exception": false,
     "start_time": "2021-11-25T16:52:29.841739",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 122)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "023bcc01",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:29.953737Z",
     "iopub.status.busy": "2021-11-25T16:52:29.952736Z",
     "iopub.status.idle": "2021-11-25T16:52:29.971187Z",
     "shell.execute_reply": "2021-11-25T16:52:29.971763Z"
    },
    "papermill": {
     "duration": 0.052026,
     "end_time": "2021-11-25T16:52:29.971763",
     "exception": false,
     "start_time": "2021-11-25T16:52:29.919737",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48744"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.TARGET.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0b055392",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:30.038510Z",
     "iopub.status.busy": "2021-11-25T16:52:30.037475Z",
     "iopub.status.idle": "2021-11-25T16:52:30.522472Z",
     "shell.execute_reply": "2021-11-25T16:52:30.523470Z"
    },
    "papermill": {
     "duration": 0.518996,
     "end_time": "2021-11-25T16:52:30.523470",
     "exception": false,
     "start_time": "2021-11-25T16:52:30.004474",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.dropna(subset=['TARGET'], axis=\"index\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ba756523",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:30.589474Z",
     "iopub.status.busy": "2021-11-25T16:52:30.588475Z",
     "iopub.status.idle": "2021-11-25T16:52:30.602475Z",
     "shell.execute_reply": "2021-11-25T16:52:30.603471Z"
    },
    "papermill": {
     "duration": 0.05,
     "end_time": "2021-11-25T16:52:30.603471",
     "exception": false,
     "start_time": "2021-11-25T16:52:30.553471",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511, 122)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "suitable-island",
   "metadata": {
    "papermill": {
     "duration": 0.028999,
     "end_time": "2021-11-25T16:52:30.662471",
     "exception": false,
     "start_time": "2021-11-25T16:52:30.633472",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Display columns with missing data only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "thick-venezuela",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:30.726473Z",
     "iopub.status.busy": "2021-11-25T16:52:30.726473Z",
     "iopub.status.idle": "2021-11-25T16:52:31.062471Z",
     "shell.execute_reply": "2021-11-25T16:52:31.063470Z"
    },
    "papermill": {
     "duration": 0.370998,
     "end_time": "2021-11-25T16:52:31.063470",
     "exception": false,
     "start_time": "2021-11-25T16:52:30.692472",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AMT_ANNUITY - 0%\n",
      "AMT_GOODS_PRICE - 0%\n",
      "NAME_TYPE_SUITE - 0%\n",
      "OWN_CAR_AGE - 66%\n",
      "OCCUPATION_TYPE - 31%\n",
      "CNT_FAM_MEMBERS - 0%\n",
      "EXT_SOURCE_1 - 56%\n",
      "EXT_SOURCE_2 - 0%\n",
      "EXT_SOURCE_3 - 20%\n",
      "APARTMENTS_AVG - 51%\n",
      "BASEMENTAREA_AVG - 59%\n",
      "YEARS_BEGINEXPLUATATION_AVG - 49%\n",
      "YEARS_BUILD_AVG - 66%\n",
      "COMMONAREA_AVG - 70%\n",
      "ELEVATORS_AVG - 53%\n",
      "ENTRANCES_AVG - 50%\n",
      "FLOORSMAX_AVG - 50%\n",
      "FLOORSMIN_AVG - 68%\n",
      "LANDAREA_AVG - 59%\n",
      "LIVINGAPARTMENTS_AVG - 68%\n",
      "LIVINGAREA_AVG - 50%\n",
      "NONLIVINGAPARTMENTS_AVG - 69%\n",
      "NONLIVINGAREA_AVG - 55%\n",
      "APARTMENTS_MODE - 51%\n",
      "BASEMENTAREA_MODE - 59%\n",
      "YEARS_BEGINEXPLUATATION_MODE - 49%\n",
      "YEARS_BUILD_MODE - 66%\n",
      "COMMONAREA_MODE - 70%\n",
      "ELEVATORS_MODE - 53%\n",
      "ENTRANCES_MODE - 50%\n",
      "FLOORSMAX_MODE - 50%\n",
      "FLOORSMIN_MODE - 68%\n",
      "LANDAREA_MODE - 59%\n",
      "LIVINGAPARTMENTS_MODE - 68%\n",
      "LIVINGAREA_MODE - 50%\n",
      "NONLIVINGAPARTMENTS_MODE - 69%\n",
      "NONLIVINGAREA_MODE - 55%\n",
      "APARTMENTS_MEDI - 51%\n",
      "BASEMENTAREA_MEDI - 59%\n",
      "YEARS_BEGINEXPLUATATION_MEDI - 49%\n",
      "YEARS_BUILD_MEDI - 66%\n",
      "COMMONAREA_MEDI - 70%\n",
      "ELEVATORS_MEDI - 53%\n",
      "ENTRANCES_MEDI - 50%\n",
      "FLOORSMAX_MEDI - 50%\n",
      "FLOORSMIN_MEDI - 68%\n",
      "LANDAREA_MEDI - 59%\n",
      "LIVINGAPARTMENTS_MEDI - 68%\n",
      "LIVINGAREA_MEDI - 50%\n",
      "NONLIVINGAPARTMENTS_MEDI - 69%\n",
      "NONLIVINGAREA_MEDI - 55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FONDKAPREMONT_MODE - 68%\n",
      "HOUSETYPE_MODE - 50%\n",
      "TOTALAREA_MODE - 48%\n",
      "WALLSMATERIAL_MODE - 51%\n",
      "EMERGENCYSTATE_MODE - 47%\n",
      "OBS_30_CNT_SOCIAL_CIRCLE - 0%\n",
      "DEF_30_CNT_SOCIAL_CIRCLE - 0%\n",
      "OBS_60_CNT_SOCIAL_CIRCLE - 0%\n",
      "DEF_60_CNT_SOCIAL_CIRCLE - 0%\n",
      "DAYS_LAST_PHONE_CHANGE - 0%\n",
      "AMT_REQ_CREDIT_BUREAU_HOUR - 14%\n",
      "AMT_REQ_CREDIT_BUREAU_DAY - 14%\n",
      "AMT_REQ_CREDIT_BUREAU_WEEK - 14%\n",
      "AMT_REQ_CREDIT_BUREAU_MON - 14%\n",
      "AMT_REQ_CREDIT_BUREAU_QRT - 14%\n",
      "AMT_REQ_CREDIT_BUREAU_YEAR - 14%\n"
     ]
    }
   ],
   "source": [
    "# if it's a larger dataset and the visualization takes too long can do this.\n",
    "# % of missing.\n",
    "for col in df.columns:\n",
    "    pct_missing = np.mean(df[col].isnull())\n",
    "    if pct_missing > 0.0: # show only the columns with missing data\n",
    "        print('{} - {}%'.format(col, round(pct_missing*100)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "presidential-domestic",
   "metadata": {
    "papermill": {
     "duration": 0.029037,
     "end_time": "2021-11-25T16:52:31.123508",
     "exception": false,
     "start_time": "2021-11-25T16:52:31.094471",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Display number of missing values per observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "diagnostic-roommate",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:31.191472Z",
     "iopub.status.busy": "2021-11-25T16:52:31.191472Z",
     "iopub.status.idle": "2021-11-25T16:52:32.620558Z",
     "shell.execute_reply": "2021-11-25T16:52:32.619555Z"
    },
    "papermill": {
     "duration": 1.467048,
     "end_time": "2021-11-25T16:52:32.620558",
     "exception": false,
     "start_time": "2021-11-25T16:52:31.153510",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "created missing indicator for: AMT_ANNUITY\n",
      "created missing indicator for: AMT_GOODS_PRICE\n",
      "created missing indicator for: NAME_TYPE_SUITE\n",
      "created missing indicator for: OWN_CAR_AGE\n",
      "created missing indicator for: OCCUPATION_TYPE\n",
      "created missing indicator for: CNT_FAM_MEMBERS\n",
      "created missing indicator for: EXT_SOURCE_1\n",
      "created missing indicator for: EXT_SOURCE_2\n",
      "created missing indicator for: EXT_SOURCE_3\n",
      "created missing indicator for: APARTMENTS_AVG\n",
      "created missing indicator for: BASEMENTAREA_AVG\n",
      "created missing indicator for: YEARS_BEGINEXPLUATATION_AVG\n",
      "created missing indicator for: YEARS_BUILD_AVG\n",
      "created missing indicator for: COMMONAREA_AVG\n",
      "created missing indicator for: ELEVATORS_AVG\n",
      "created missing indicator for: ENTRANCES_AVG\n",
      "created missing indicator for: FLOORSMAX_AVG\n",
      "created missing indicator for: FLOORSMIN_AVG\n",
      "created missing indicator for: LANDAREA_AVG\n",
      "created missing indicator for: LIVINGAPARTMENTS_AVG\n",
      "created missing indicator for: LIVINGAREA_AVG\n",
      "created missing indicator for: NONLIVINGAPARTMENTS_AVG\n",
      "created missing indicator for: NONLIVINGAREA_AVG\n",
      "created missing indicator for: APARTMENTS_MODE\n",
      "created missing indicator for: BASEMENTAREA_MODE\n",
      "created missing indicator for: YEARS_BEGINEXPLUATATION_MODE\n",
      "created missing indicator for: YEARS_BUILD_MODE\n",
      "created missing indicator for: COMMONAREA_MODE\n",
      "created missing indicator for: ELEVATORS_MODE\n",
      "created missing indicator for: ENTRANCES_MODE\n",
      "created missing indicator for: FLOORSMAX_MODE"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "created missing indicator for: FLOORSMIN_MODE\n",
      "created missing indicator for: LANDAREA_MODE\n",
      "created missing indicator for: LIVINGAPARTMENTS_MODE\n",
      "created missing indicator for: LIVINGAREA_MODE\n",
      "created missing indicator for: NONLIVINGAPARTMENTS_MODE\n",
      "created missing indicator for: NONLIVINGAREA_MODE\n",
      "created missing indicator for: APARTMENTS_MEDI\n",
      "created missing indicator for: BASEMENTAREA_MEDI\n",
      "created missing indicator for: YEARS_BEGINEXPLUATATION_MEDI\n",
      "created missing indicator for: YEARS_BUILD_MEDI\n",
      "created missing indicator for: COMMONAREA_MEDI\n",
      "created missing indicator for: ELEVATORS_MEDI\n",
      "created missing indicator for: ENTRANCES_MEDI\n",
      "created missing indicator for: FLOORSMAX_MEDI\n",
      "created missing indicator for: FLOORSMIN_MEDI\n",
      "created missing indicator for: LANDAREA_MEDI\n",
      "created missing indicator for: LIVINGAPARTMENTS_MEDI\n",
      "created missing indicator for: LIVINGAREA_MEDI\n",
      "created missing indicator for: NONLIVINGAPARTMENTS_MEDI\n",
      "created missing indicator for: NONLIVINGAREA_MEDI\n",
      "created missing indicator for: FONDKAPREMONT_MODE\n",
      "created missing indicator for: HOUSETYPE_MODE\n",
      "created missing indicator for: TOTALAREA_MODE\n",
      "created missing indicator for: WALLSMATERIAL_MODE\n",
      "created missing indicator for: EMERGENCYSTATE_MODE\n",
      "created missing indicator for: OBS_30_CNT_SOCIAL_CIRCLE\n",
      "created missing indicator for: DEF_30_CNT_SOCIAL_CIRCLE\n",
      "created missing indicator for: OBS_60_CNT_SOCIAL_CIRCLE\n",
      "created missing indicator for: DEF_60_CNT_SOCIAL_CIRCLE\n",
      "created missing indicator for: DAYS_LAST_PHONE_CHANGE\n",
      "created missing indicator for: AMT_REQ_CREDIT_BUREAU_HOUR\n",
      "created missing indicator for: AMT_REQ_CREDIT_BUREAU_DAY\n",
      "created missing indicator for: AMT_REQ_CREDIT_BUREAU_WEEK\n",
      "created missing indicator for: AMT_REQ_CREDIT_BUREAU_MON\n",
      "created missing indicator for: AMT_REQ_CREDIT_BUREAU_QRT\n",
      "created missing indicator for: AMT_REQ_CREDIT_BUREAU_YEAR\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEJCAYAAAB7UTvrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAfZUlEQVR4nO3de5hU1Znv8e/LXQMCQqtIGxtjZ4LAEREVL0GCiqhR8HpwzvHCQUkMKmPGmSExR+JtJGOiGSckz6MREY1Ro2ci8e54iXflDo2gdJBLixObi0qiEIH3/LFWbTbV1d3VdHV3dfP7PM9+uuqttfdau2rVfvdl1W5zd0RERADatXQDRESkeCgpiIhIQklBREQSSgoiIpJQUhARkYSSgoiIJDq0dAN2V+/evb2srKylmyEi0mrMmzdvvbuX1FWm1SaFsrIy5s6d29LNEBFpNcxsdX1ldPpIREQSSgoiIpJQUhARkUSrvaaQy5dffklVVRVbtmxp6aZI1KVLF0pLS+nYsWNLN0VE8tCmkkJVVRXdunWjrKwMM2vp5uzx3J0NGzZQVVVFv379Wro5IpKHNnX6aMuWLfTq1UsJoUiYGb169dKRm0gr0qaSAqCEUGT0eYi0Lm0uKYiIyO5rU9cUspVNebKgy1s17YyCLq85HHfccbzxxhsNmuf6669n+PDhnHzyyU3UKpHCSn/XW+P3tJi06aQgNDghANx4441N0BIRaQ10+qiAVq1aRf/+/bn88ssZMGAAo0aN4osvvmDEiBHJLTnWr19P5p5NM2fOZOzYsZx55pn069ePX/ziF9x+++0cccQRDBs2jI0bN9Za14gRI7jmmmsYPnw4/fv3Z86cOZxzzjmUl5fzox/9KCnXtWtXAD766COGDx/O4MGDGThwIK+++irbt2/n0ksvZeDAgQwaNIg77rgDgEsvvZRHH30UCLcTmTp1KkOGDGHQoEEsX74cgOrqak455RSGDBnCd77zHQ4++GDWr19f8PdURJqXkkKBrVixgkmTJrF06VJ69OjBY489Vmf5iooKHnzwQd555x2uu+469t57bxYsWMCxxx7LrFmz6py3U6dOvPLKK3z3u99lzJgxTJ8+nYqKCmbOnMmGDRt2Kfvggw9y6qmnsnDhQhYtWsTgwYNZuHAhH374IRUVFSxZsoTx48fnrKd3797Mnz+fK664gp/+9KcA3HDDDYwcOZL58+dz9tlns2bNmga8SyJSrJQUCqxfv34MHjwYgCOPPJJVq1bVWf5b3/oW3bp1o6SkhO7du3PmmWcCMGjQoHrnPeuss5KyAwYMoE+fPnTu3JlDDjmEtWvX7lL2qKOO4t577+XHP/4xS5YsoVu3bhxyyCGsXLmSq666imeeeYZ99tknZz3nnHNOjfV57bXXGDduHACjR4+mZ8+edbZVRFoHJYUC69y5c/K4ffv2bNu2jQ4dOrBjxw6AGmP20+XbtWuXPG/Xrh3btm3Lq670fLXNO3z4cF555RX69u3LRRddxKxZs+jZsyeLFi1ixIgRTJ8+ncsuu6zOejLrA+GHaSLS9igpNIOysjLmzZsHkJyrb26rV69mv/324/LLL2fChAnMnz+f9evXs2PHDs4991xuuukm5s+fn/fyTjjhBB555BEAnnvuOTZt2tRUTReRZtSmRx8Vy9C0a6+9lgsuuID777+fkSNHtkgbXn75ZW677TY6duxI165dmTVrFh9++CHjx49PjmJuvfXWvJc3depULrzwQh5++GFOPPFE+vTpQ7du3Zqq+SLSTKy1ngYYOnSoZ/+TnWXLltG/f/8WatGeZevWrbRv354OHTrw5ptvcsUVV7Bw4cKcZfW5SFPT7xTyY2bz3H1oXWXa9JGCNJ01a9ZwwQUXsGPHDjp16sTdd9/d0k0SqSGTLJQo8qekUOQmTZrE66+/vkts8uTJtQ4fbS7l5eUsWLCgRdsgIoWnpFDkpk+f3tJNEJE9SJsbfdRar5G0Vfo8RFqXNpUUunTpwoYNG7QhKhKZf7LTpUuXlm6KiOSpTZ0+Ki0tpaqqiurq6pZuikSZf8cpIq1Dm0oKHTt21L99FBFphDZ1+khERBpHSUFERBJ5JwUza29mC8zsifi8n5m9bWYrzOxhM+sU453j88r4ellqGT+I8ffM7NRUfHSMVZrZlMKtnoiINERDjhQmA8tSz38C3OHu5cAmYEKMTwA2ufuhwB2xHGZ2GDAOGACMBn4ZE017YDpwGnAYcGEsKyIizSyvpGBmpcAZwK/jcwNGAplbft4HjI2Px8TnxNdPiuXHAA+5+1Z3/wCoBI6OU6W7r3T3vwEPxbIiItLM8j1S+Dnwz8CO+LwX8Im7Z27aXwX0jY/7AmsB4uufxvJJPGue2uI1mNlEM5trZnM17FREpPDqTQpm9m3gY3eflw7nKOr1vNbQeM2g+13uPtTdh5aUlNTRahER2R35/E7heOAsMzsd6ALsQzhy6GFmHeLRQCmwLpavAg4CqsysA9Ad2JiKZ6TnqS0uIiLNqN4jBXf/gbuXunsZ4ULxi+7+v4CXgPNisUuAx+Pj2fE58fUXPdx3YjYwLo5O6geUA+8Ac4DyOJqpU6xjdkHWTkREGqQxv2j+F+AhM7sZWADcE+P3APebWSXhCGEcgLsvNbNHgHeBbcAkd98OYGZXAs8C7YEZ7r60Ee0SEZHd1KCk4O4vAy/HxysJI4eyy2wBzq9l/luAW3LEnwKeakhbRESk8PSLZhERSSgpiIhIQklBREQSSgoiIpJQUhARkYSSgoiIJJQUREQkoaQgIiIJJQUREUkoKYiISEJJQUREEkoKIiKSUFIQEZGEkoKIiCSUFEREJKGkICIiCSUFERFJKCmIiEhCSUFERBJKCiIiklBSEBGRhJKCiIgklBRERCShpCAiIgklBRERSSgpiIhIQklBREQSSgoiIpJQUhARkYSSgoiIJJQUREQkoaQgIiIJJQUREUkoKYiISEJJQUREEkoKIiKSUFIQEZFEvUnBzLqY2TtmtsjMlprZDTHez8zeNrMVZvawmXWK8c7xeWV8vSy1rB/E+HtmdmoqPjrGKs1sSuFXU0RE8pHPkcJWYKS7Hw4MBkab2TDgJ8Ad7l4ObAImxPITgE3ufihwRyyHmR0GjAMGAKOBX5pZezNrD0wHTgMOAy6MZUVEpJnVmxQ8+Et82jFODowEHo3x+4Cx8fGY+Jz4+klmZjH+kLtvdfcPgErg6DhVuvtKd/8b8FAsKyIizSyvawpxj34h8DHwPPAn4BN33xaLVAF94+O+wFqA+PqnQK90PGue2uIiItLM8koK7r7d3QcDpYQ9+/65isW/VstrDY3XYGYTzWyumc2trq6uv+EiItIgDRp95O6fAC8Dw4AeZtYhvlQKrIuPq4CDAOLr3YGN6XjWPLXFc9V/l7sPdfehJSUlDWm6iIjkIZ/RRyVm1iM+3gs4GVgGvAScF4tdAjweH8+Oz4mvv+juHuPj4uikfkA58A4wByiPo5k6ES5Gzy7EyomISMN0qL8IfYD74iihdsAj7v6Emb0LPGRmNwMLgHti+XuA+82sknCEMA7A3Zea2SPAu8A2YJK7bwcwsyuBZ4H2wAx3X1qwNRQRkbzVmxTcfTFwRI74SsL1hez4FuD8WpZ1C3BLjvhTwFN5tFdERJqQftEsIiIJJQUREUkoKYiISEJJQUREEkoKIiKSUFIQEZGEkoKIiCSUFEREJKGkICIiCSUFERFJKCmIiEhCSUFERBJKCiIiklBSEBGRhJKCiIgklBRERCShpCAiIgklBRERSSgpiIhIot7/0dzalE15Mnm8atoZLdgSEZHWR0cKIiKSUFIQEZGEkoKIiCT2mKRQNuXJXa43iIhITXtMUhARkfopKYiISEJJQUREEkoKIiKSUFIQEZGEkoKIiCSUFEREJKGkICIiCSUFERFJtLm7pIqI1EV3Uq6bjhRERCShpCAiIgklBRERSSgpiIhIot6kYGYHmdlLZrbMzJaa2eQY39fMnjezFfFvzxg3M7vTzCrNbLGZDUkt65JYfoWZXZKKH2lmS+I8d5qZNcXKiohI3fI5UtgG/KO79weGAZPM7DBgCvCCu5cDL8TnAKcB5XGaCPwKQhIBpgLHAEcDUzOJJJaZmJpvdONXTUREGqrepODuH7n7/Ph4M7AM6AuMAe6Lxe4DxsbHY4BZHrwF9DCzPsCpwPPuvtHdNwHPA6Pja/u4+5vu7sCs1LJERKQZNeiagpmVAUcAbwP7u/tHEBIHsF8s1hdYm5qtKsbqilfliOeqf6KZzTWzudXV1Q1puoiI5CHvpGBmXYHHgH9w98/qKpoj5rsRrxl0v8vdh7r70JKSkvqaLCIiDZRXUjCzjoSE8Bt3/38x/Od46of49+MYrwIOSs1eCqyrJ16aIy4iIs0sn9FHBtwDLHP321MvzQYyI4guAR5PxS+Oo5CGAZ/G00vPAqPMrGe8wDwKeDa+ttnMhsW6Lk4tS0REmlE+9z46HrgIWGJmC2Psh8A04BEzmwCsAc6Prz0FnA5UAp8D4wHcfaOZ3QTMieVudPeN8fEVwExgL+DpOImISDOrNym4+2vkPu8PcFKO8g5MqmVZM4AZOeJzgYH1tUVERJqWftEsIiIJJQUREUkoKYiISEJJQUREEkoKIiKSUFIQEZGEkoKIiCSUFEREJKGkICIiCSUFERFJKCmIiEhCSUFERBJKCiLSqpRNeZKyKU+2dDPaLCUFERFJKCmIiEhCSUFERBJKCiIiklBSEBGRhJKCiIgklBRERCShpCAiIgklBRERSSgpiIhIQklBREQSSgoiIpLo0NINaEnpm2qtmnZGC7ZERKQ46EhBREQSSgoiIpJQUhARkYSSgoiIJJQUREQkoaQgIiIJJQUREUkoKYiISEJJQUREEkoKIiKSUFIQEZFEvUnBzGaY2cdmVpGK7Wtmz5vZivi3Z4ybmd1pZpVmttjMhqTmuSSWX2Fml6TiR5rZkjjPnWZmhV5JERHJTz5HCjOB0VmxKcAL7l4OvBCfA5wGlMdpIvArCEkEmAocAxwNTM0kklhmYmq+7LpERKSZ1JsU3P0VYGNWeAxwX3x8HzA2FZ/lwVtADzPrA5wKPO/uG919E/A8MDq+to+7v+nuDsxKLUtERJrZ7l5T2N/dPwKIf/eL8b7A2lS5qhirK16VIy4iIi2g0Beac10P8N2I51642UQzm2tmc6urq3eziSIiUpvdTQp/jqd+iH8/jvEq4KBUuVJgXT3x0hzxnNz9Lncf6u5DS0pKdrPpIiJSm91NCrOBzAiiS4DHU/GL4yikYcCn8fTSs8AoM+sZLzCPAp6Nr202s2Fx1NHFqWWJiEgzq/ffcZrZb4ERQG8zqyKMIpoGPGJmE4A1wPmx+FPA6UAl8DkwHsDdN5rZTcCcWO5Gd89cvL6CMMJpL+DpOImISAuoNym4+4W1vHRSjrIOTKplOTOAGTnic4GB9bVDRESann7RLCIiiXqPFIpZ2ZQnk8erpp3Rgi0REWkbdKQgIiIJJQUREUm06tNHIiKtTbGf9taRgoiIJHSkICLSRIr9qCAXHSmIiEhCSUFERBJKCiIiklBSEBGRhJKC5K1sypO7XDgTkbZHo4+kUVrj6AoRqZ2OFEREJKGkICIiCSUFERFJKCmIiEhCSUFERBJKCiIiktCQ1GZS29DNTFzDOUWkGOhIQUREEkoKIiKSUFLIQbdzaDy9hyKtk5KCtHpKQCKFo6QgIlIEimXnRklBREQSGpIqzaaxd1TVHVlFmp6SQhHakzZ+TbWue9J7KFJISgp50kZGRPYESgpSQzEkQP3SW6RlKCk0AW3QilMxJDuRYqfRRyIiBVAsQ0obS0cKjaA9z7ZBR3YiOykptBJKQM1Ld7Xds+j7tZOSgkgj5dqgaCMjrZWuKbRybeU8pogUBx0ptEE69VG8cn0GOqqQYqKksIfTBql4tXQS12mxPVPRJAUzGw38O9Ae+LW7T2vhJokUnYZslBuSVFo6AUnxKIqkYGbtgenAKUAVMMfMZrv7uy3bMpHitzuJIp+yu9MG3b+q9SuKpAAcDVS6+0oAM3sIGAMoKYi0Ug25tlWsCaA1tbVQzN1bug2Y2XnAaHe/LD6/CDjG3a/MKjcRmBif/h3wHtAbWJ9jsbni+caaqqzqUl2qS3W1ZF0Hu3tJjtd3cvcWn4DzCdcRMs8vAv4jz3nn5hvPN9ZUZVWX6lJdqqtY6qptKpbfKVQBB6WelwLrWqgtIiJ7rGJJCnOAcjPrZ2adgHHA7BZuk4jIHqcoLjS7+zYzuxJ4ljAkdYa7L81z9rsaEM831lRlVZfqUl2qq1jqyqkoLjSLiEhxKJbTRyIiUgSUFEREJKGkICIiiaK40NwQZvYNwq+d+wJOGLo6292X1VK2L/C2u/8lxo4Ghrn7nWZ2GDAaWO7uT6Xmm+XuF2ct6wTgPOBld/+9me0FTAGGAHsBV3vqthypUVTr3P2/zOzvgeOAauCL2K5twArgt+7+aQHenj2Wme3n7h/nUa6Xu29ojjbJnqdN9MN8f9BQDBPwL8BCwsb4f8dpSiaWVfZqwi+e5wOrCIlkKvAW8FfgVuBFYDmwAVhGGAb7B+AvwCZCsgG4PNbxMfB6rPMu4OfACcAWwob+VeB7QAnwG+DhuLz7gf+Mf9cBC4A3gF8CtxBu5zGimd7D/RpQtlcj6+oOTEu9x5n3eRrQI6vsPsCf4nv09zF2APArYAnQC/hxfPx7oD+wb5x6AX8Gesbn3YF74uf1GLA/MBRYCVQCW4FfA19L1T8UeAl4gPCbmeeBT4F58bNeGp9Xxz40E3gGWAwsAp4Gvgt0zFqv9rFf3AQcH2N7A/8c5+8CXBr73r8BXbPmX5V63BH4UazzjricQ4FXgE+AjbFvdk3NcwgwA7gZ6ArcDVQAvwP+CXgytn9e7K8/a+r1At4H/kcTrdccwvc6s04PAWfQNvvhnPj51btecdlP5/W9be4NeyM3Mu9nd84Y7wSsyIotiZ1lDVAGzCVskNsTNvCfxQ4wH/ht7AgnAiOAjwh78CfGZc0hbOiXAV+Jy56fqmtBXOao2Amqgc3AJUCP2FHax/naEzr/3oSjDoABsUy+nfZW4MNMp0113ArCjQUzHffdrI5bW6ddHDtb/1THXElIdp8AF2a1obaO+2F8HzOd9hPgEeCA1LynEO6G+xbhKCszvUBIxmMJG5LHCEOUr4rLXUzYKfgqsIOQ2D9ITR7/riR80W6O7+c18T14CTgqtmEt8N+EvvFOLLMAOA24ML5+Xiz7OuELXAp8H/i/hET/HiExlMZpFGEj9Z/s3EjsS9i4fA78A+GLfXt8T34W36MXgF8Aw4G/xekzQv/ZHNdrc4z9LNb5AWHjOYuwUTg7trWacCuDjbGOswkb7isIG9UK4B/jZ/ZaXM4JhJ2bGwkb06pYT6HW63PCxu9vqXXaHqfPYrsLtV4PA4/H9mTW6RTC9+lx2l4/fDWu67/nsV5HAh+1xaSwnHDvjuz4MsIGbHFq2kLYCG+NZboSvli3EzbgC2K8XfwwNgODY2wlYU+jJ2FDOjfGfweMjx/cvcDQGF8KzEm1pyOwOnbS9XHZ+8b27BPb2xOYF8s/GztHutMeUEenvZfwBcl02s6EvbO1hC9JpuPuIHwh0h03V6c9mJAwfx/rfgk4Kpa7h/ClznTaA+PjXTou4Uv3k9hJM512JXAf8K+p9dpO2Ph8HuvJTJuBL1LlriN8OXsREvea1GvXxs9yUCq2NfV4Yaq/dIif91up1+cDS+LjbxKO2L6M7ZiYVdciYl9J7SC8R+g3y7PWa2VcTua9Xknc0McyHQh7e5/Ez2xB/NwzQ8P/g3CEun9t60XoW+8BFj/ndL9bEGPdCLeKeSq2517Cxj29Xouz1uutuNzOwLJCrVdcp1nAu6llfpBVd0HWC1icKZ9Zp/j3/fQ67QH9MNd6vZRerzq3sy29oW/IRDj/X0k4pL0rTs8Qzs1/j7Bxy0yvA6cTzuln5n+bcFpnO9AuFe9O2GD/jrB3s4Zwymll6ktwQCz3AGEj+Xb8AFfGjnR4Vluvia+tIZzKeoFwyugLwlHLcmB8LFsJvJJjfbcTNujZnTb5gGOnfZ3wpZkfY5kvybXx/Ul3mhqdNtVxM50482XKLG9JqtP+d2zDxKy6FmV9IecAzxGS04pUPcsIpxP+K2tdlwFrs2JrCAl3NXBzjvK/IyT5brEPfJ+wx7iSsHG5KrbhT4Qjp58T9sg/Au7PWt6bwA/je7saGBvjFZn3DziTkMDfItyv673U/CsIffDtrOUuz7FeH8XPbAXhh5rp194nfKmvJnzhvwTOAc4lbtgIpxxnxvfgh4S99a8S+uwTWctbCFwfP4/17NyRSa/XEMKpmsx6vVvI9SLspW5OrdPKOJ1dyPUi7JCMJHwXhhC/U7EP/Jldk21b6IfPEU7XVda3XvG1tdmxnNvZpth4N+UUO9Ww2JnOi49nACdklSslbMgfTMU6x7/HZ5XtTcz2hPOP/1pH/XsDA4HDY2ffH/h6LWUPBA6Mj3vE9v7P+PcbqXKZDzfdafcnbIBfz9EJ26U/YMJpqi3A6vj85qz34dO6Om0sdxXhizsy1XHfB25Id1zC6a9lsVOen+m4hIR3JSHhZTptT8LRw1bCHvBGwumJu4F9s9br34Drs2I3xmVnnxo8FHg09QV5K7Z9amoqia+fQ9ioLCAkt6cIG5Lsc+SHxzY/DXyDcJT2CSFhvxsfv0a4O28Z4VTA5vgevR8fPwP0y1ruA8Avs2K/BiYBX2bFvxbraEfYgL5K2OO7NzXtH8tOjp9r5kj0XcLGq3vWMk8i7FEuI5wqeoyw0d5EOC3zPmHH55jUen0e4ysKvF6ZdVpH2PgXer2qCDtS6+I6DUv1l7cJiazY++FgavbDTdTsh18nfL/uJCS8OtcrLntsXtvYptyAa8pvYufGc3n8YDfGzv574vnHrE57cvYHTDgCqsyx7EOBR/PotAfEzvhwquOuIxzGdshaZq4N6GeEjcnmTKeNZY8lHH2lLxJ+g6wLhzF+WfyyN6gsYfTXwFxlG1hX/1rKXh7f83TZYwgbuV6EjdK1hD3bzPniwwjJ93TC/wvJjucqe0ZW2W8S9oavqWf+AYQkX1td12SVvTaWPTZXe+PzXoSdpQdy9KlZtfTjGvFMjJ07H32ADQ2Y//4GlH2CkFB711Pum/H9GpWKnRDXf1RW2RrxOsp+k3DBvM6yuerfzbI/Z+eRxN6E5PUa4bpMd8L34kbCNbCfkJVYa5t0m4siZ2bj3f3efOL1lY3DaL/m7hW7M39Dy5rZ1YQ9x78SrqlMJpzam0Q4EvoEmOzuj5vZVcBthL3SwbHsVwlHH9llryYkxzrLNrCugwmnSQ7Io67BhGsqgwh7akcTThMMJSTH+wlJ42Xg/xA2VB8TLsgfE9+qfMrmWm5tsT/mWVdtZY8mDJMuibEVcd7T2Xl//ndi3d8inGfPxIjx03KUzY5BOBrNnj9XXbWVrW25tc3/JfCqu59lZpcR+kQJ4Sj3D4TrbpMIF9OvBW5x92lmdnmM986z7Peylrue0Cd7Z8UmEXb48qnreznKXhaXeyDhc/oDYTTWXwlHNLMIfXM9YUftUcIO0OHufg71ySdzaGq5idTFpvrijS1b6LqofQRYV8LRSCY2OZbNXJcoVNmmrCt7FNsSdo5M+wzYJ85Twc7RZgUp24R15RqJ9yXh9OZkao7Oey4+r6tsY+cvVF0nxvXOjCRckHpf5rDziHkROy/+FqRsE9ZVYzQkO6/PLCQ1QjITy2eb0+p+vNYWmdniWl4qBzpnvV4e/+aK51O2sfM3tK43CeeLV5nZCMJ1khsJpxMysUcJez1rAQpYtqnqWkc40gD4k7t/Zmbb3P2vZrY9E4uvfxmX+bmZFaRsE9Y1lLBB/TbwqbsvNLO1hNOE3wb+GGNfEK6tTCYMdPinOso2dv5C1PU5sNjMesV1rjazdoSh7J6JsZMXsmwT1lUBXEC4Trgofn4VZjYlfr4VZjbU3eea2ddjrH75ZA5NTTsRLhQNZtfRUwcTLgR+nCN2epxnd8o2dv6GlM01AuwlwuHu9lSsA2GPbnvW+9LYsk1VV41RbDF2AGFvOz2ybS6p4c+FKNtUdcXHNUbixXhpdqy2eL6xpiqbHSP3SMJV8fnWTCzOuzrGCla2CevKNRryA8IOzlp2HSH5R7JGSNa6PWrpDaImh/BbgBNqiT+fqyypUVUNKdvY+RtYV64RYJlY9giwUuDMHLHdLtuEddUYxUYYn5+MYkvFD8wRa1TZpqorxusciZcr1pCyjZ2/EHWlXt+bmqOqasSaqmwB69plNGSMd8uO5TvpQrOIiCR0l1QREUkoKYiISEJJQaQOZvZGA8uPMLMnmqo9Ik1NSUGkDu5+XEu3QaQ5KSmI1MHMMv+caYSZvWxmj5rZcjP7jZlZfG10jL1GuMdNZt6vmNkMM5tjZgvMbEyMf9/MZsTHg8yswsz2boHVE6lBSUEkf0cQ7jl0GOG2AsebWRfCDcjOJNyn5oBU+euAF939KMKtIW4zs68Q7llzqJmdTbgZ3Hfc/fPmWw2R2ikpiOTvHXevcvcdhNsIlBFuuPeBu6/wML77gVT5UcAUM1tIuLdRF+Crcf5LCfci+qO7v958qyBSN93mQiR/W1OPt7Pz+1Pbj30MONfd38vxWjnhRm8HFq55Io2nIwWRxlkO9DOzr8XnF6Zeexa4KnXt4Yj4tzvhduPDgV5mdl4ztlekTkoKIo3g7lsI/3PiyXiheXXq5ZsI/2Zycbx52U0xfgfhH9S8D0wAppnZfs3YbJFa6TYXIiKS0JGCiIgklBRERCShpCAiIgklBRERSSgpiIhIQklBREQSSgoiIpJQUhARkcT/B9oGlNtA9RKNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def missingplot(df):\n",
    "    # first create missing indicator for features with missing data\n",
    "    for col in df.columns:\n",
    "        missing = df[col].isnull()\n",
    "        num_missing = np.sum(missing)\n",
    "        \n",
    "        if num_missing > 0:  \n",
    "            print('created missing indicator for: {}'.format(col))\n",
    "            df['{}_ismissing'.format(col)] = missing\n",
    "\n",
    "\n",
    "    # then based on the indicator, plot the histogram of missing values\n",
    "    ismissing_cols = [col for col in df.columns if 'ismissing' in col]\n",
    "    df['num_missing'] = df[ismissing_cols].sum(axis=1)\n",
    "\n",
    "    df['num_missing'].value_counts().reset_index().sort_values(by='index').plot.bar(x='index', y='num_missing')\n",
    "    \n",
    "missingplot(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "controlled-status",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:32.689555Z",
     "iopub.status.busy": "2021-11-25T16:52:32.687557Z",
     "iopub.status.idle": "2021-11-25T16:52:35.690504Z",
     "shell.execute_reply": "2021-11-25T16:52:35.689517Z"
    },
    "papermill": {
     "duration": 3.036948,
     "end_time": "2021-11-25T16:52:35.690504",
     "exception": false,
     "start_time": "2021-11-25T16:52:32.653556",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# drop rows with more than 52 missing values\n",
    "for i in range(52, 63):\n",
    "    ind_missing = df[df['num_missing'] == i].index\n",
    "    df = df.drop(ind_missing, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "comparative-monitoring",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:35.761519Z",
     "iopub.status.busy": "2021-11-25T16:52:35.760485Z",
     "iopub.status.idle": "2021-11-25T16:52:36.322904Z",
     "shell.execute_reply": "2021-11-25T16:52:36.323900Z"
    },
    "papermill": {
     "duration": 0.600381,
     "end_time": "2021-11-25T16:52:36.323900",
     "exception": false,
     "start_time": "2021-11-25T16:52:35.723519",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# drop rows with missing values\n",
    "for i in range(48, 50):\n",
    "    ind_missing = df[df['num_missing'] == i].index\n",
    "    df = df.drop(ind_missing, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "special-cameroon",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:36.394902Z",
     "iopub.status.busy": "2021-11-25T16:52:36.393902Z",
     "iopub.status.idle": "2021-11-25T16:52:36.769912Z",
     "shell.execute_reply": "2021-11-25T16:52:36.768912Z"
    },
    "papermill": {
     "duration": 0.414009,
     "end_time": "2021-11-25T16:52:36.769912",
     "exception": false,
     "start_time": "2021-11-25T16:52:36.355903",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# drop rows with missing values\n",
    "for i in range(1, 3):\n",
    "    ind_missing = df[df['num_missing'] == i].index\n",
    "    df = df.drop(ind_missing, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "instant-queue",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:36.840912Z",
     "iopub.status.busy": "2021-11-25T16:52:36.839911Z",
     "iopub.status.idle": "2021-11-25T16:52:37.006913Z",
     "shell.execute_reply": "2021-11-25T16:52:37.007912Z"
    },
    "papermill": {
     "duration": 0.204001,
     "end_time": "2021-11-25T16:52:37.007912",
     "exception": false,
     "start_time": "2021-11-25T16:52:36.803911",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# drop rows missing values\n",
    "ind_missing = df[df['num_missing'] == 47].index\n",
    "df = df.drop(ind_missing, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "rolled-preservation",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:37.076911Z",
     "iopub.status.busy": "2021-11-25T16:52:37.075913Z",
     "iopub.status.idle": "2021-11-25T16:52:37.228913Z",
     "shell.execute_reply": "2021-11-25T16:52:37.227910Z"
    },
    "papermill": {
     "duration": 0.189002,
     "end_time": "2021-11-25T16:52:37.228913",
     "exception": false,
     "start_time": "2021-11-25T16:52:37.039911",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# drop rows missing values\n",
    "ind_missing = df[df['num_missing'] == 50].index\n",
    "df = df.drop(ind_missing, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "desperate-telling",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:37.296913Z",
     "iopub.status.busy": "2021-11-25T16:52:37.295911Z",
     "iopub.status.idle": "2021-11-25T16:52:37.431945Z",
     "shell.execute_reply": "2021-11-25T16:52:37.432952Z"
    },
    "papermill": {
     "duration": 0.173041,
     "end_time": "2021-11-25T16:52:37.432952",
     "exception": false,
     "start_time": "2021-11-25T16:52:37.259911",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# drop rows missing values\n",
    "ind_missing = df[df['num_missing'] == 3].index\n",
    "df = df.drop(ind_missing, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "naked-cleveland",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:37.501946Z",
     "iopub.status.busy": "2021-11-25T16:52:37.501946Z",
     "iopub.status.idle": "2021-11-25T16:52:37.510946Z",
     "shell.execute_reply": "2021-11-25T16:52:37.510946Z"
    },
    "papermill": {
     "duration": 0.046973,
     "end_time": "2021-11-25T16:52:37.511948",
     "exception": false,
     "start_time": "2021-11-25T16:52:37.464975",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(98491, 190)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "available-vietnam",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:37.582949Z",
     "iopub.status.busy": "2021-11-25T16:52:37.579918Z",
     "iopub.status.idle": "2021-11-25T16:52:37.767913Z",
     "shell.execute_reply": "2021-11-25T16:52:37.767913Z"
    },
    "papermill": {
     "duration": 0.225946,
     "end_time": "2021-11-25T16:52:37.768910",
     "exception": false,
     "start_time": "2021-11-25T16:52:37.542964",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AMT_ANNUITY - 0%\n",
      "AMT_GOODS_PRICE - 0%\n",
      "NAME_TYPE_SUITE - 1%\n",
      "OWN_CAR_AGE - 64%\n",
      "OCCUPATION_TYPE - 31%\n",
      "CNT_FAM_MEMBERS - 0%\n",
      "EXT_SOURCE_1 - 52%\n",
      "EXT_SOURCE_2 - 0%\n",
      "EXT_SOURCE_3 - 27%\n",
      "APARTMENTS_AVG - 9%\n",
      "BASEMENTAREA_AVG - 33%\n",
      "YEARS_BEGINEXPLUATATION_AVG - 3%\n",
      "YEARS_BUILD_AVG - 58%\n",
      "COMMONAREA_AVG - 69%\n",
      "ELEVATORS_AVG - 17%\n",
      "ENTRANCES_AVG - 8%\n",
      "FLOORSMAX_AVG - 6%\n",
      "FLOORSMIN_AVG - 63%\n",
      "LANDAREA_AVG - 36%\n",
      "LIVINGAPARTMENTS_AVG - 64%\n",
      "LIVINGAREA_AVG - 8%\n",
      "NONLIVINGAPARTMENTS_AVG - 67%\n",
      "NONLIVINGAREA_AVG - 23%\n",
      "APARTMENTS_MODE - 9%\n",
      "BASEMENTAREA_MODE - 33%\n",
      "YEARS_BEGINEXPLUATATION_MODE - 3%\n",
      "YEARS_BUILD_MODE - 58%\n",
      "COMMONAREA_MODE - 69%\n",
      "ELEVATORS_MODE - 17%\n",
      "ENTRANCES_MODE - 8%\n",
      "FLOORSMAX_MODE - 6%\n",
      "FLOORSMIN_MODE - 63%\n",
      "LANDAREA_MODE - 36%\n",
      "LIVINGAPARTMENTS_MODE - 64%\n",
      "LIVINGAREA_MODE - 8%\n",
      "NONLIVINGAPARTMENTS_MODE - 67%\n",
      "NONLIVINGAREA_MODE - 23%\n",
      "APARTMENTS_MEDI - 9%\n",
      "BASEMENTAREA_MEDI - 33%\n",
      "YEARS_BEGINEXPLUATATION_MEDI - 3%\n",
      "YEARS_BUILD_MEDI - 58%\n",
      "COMMONAREA_MEDI - 69%\n",
      "ELEVATORS_MEDI - 17%\n",
      "ENTRANCES_MEDI - 8%\n",
      "FLOORSMAX_MEDI - 6%\n",
      "FLOORSMIN_MEDI - 63%\n",
      "LANDAREA_MEDI - 36%\n",
      "LIVINGAPARTMENTS_MEDI - 64%\n",
      "LIVINGAREA_MEDI - 8%\n",
      "NONLIVINGAPARTMENTS_MEDI - 67%\n",
      "NONLIVINGAREA_MEDI - 23%\n",
      "FONDKAPREMONT_MODE - 62%\n",
      "HOUSETYPE_MODE - 7%\n",
      "TOTALAREA_MODE - 2%\n",
      "WALLSMATERIAL_MODE - 9%\n",
      "EMERGENCYSTATE_MODE - 2%\n",
      "OBS_30_CNT_SOCIAL_CIRCLE - 1%\n",
      "DEF_30_CNT_SOCIAL_CIRCLE - 1%\n",
      "OBS_60_CNT_SOCIAL_CIRCLE - 1%\n",
      "DEF_60_CNT_SOCIAL_CIRCLE - 1%\n",
      "DAYS_LAST_PHONE_CHANGE - 0%\n",
      "AMT_REQ_CREDIT_BUREAU_HOUR - 20%\n",
      "AMT_REQ_CREDIT_BUREAU_DAY - 20%\n",
      "AMT_REQ_CREDIT_BUREAU_WEEK - 20%\n",
      "AMT_REQ_CREDIT_BUREAU_MON - 20%\n",
      "AMT_REQ_CREDIT_BUREAU_QRT - 20%\n",
      "AMT_REQ_CREDIT_BUREAU_YEAR - 20%\n"
     ]
    }
   ],
   "source": [
    "# if it's a larger dataset and the visualization takes too long can do this.\n",
    "# % of missing.\n",
    "for col in df.columns:\n",
    "    pct_missing = np.mean(df[col].isnull())\n",
    "    if pct_missing > 0.0: # show only the columns with missing data\n",
    "        print('{} - {}%'.format(col, round(pct_missing*100)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "deluxe-protection",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:37.869918Z",
     "iopub.status.busy": "2021-11-25T16:52:37.868918Z",
     "iopub.status.idle": "2021-11-25T16:52:38.023016Z",
     "shell.execute_reply": "2021-11-25T16:52:38.024012Z"
    },
    "papermill": {
     "duration": 0.218098,
     "end_time": "2021-11-25T16:52:38.024012",
     "exception": false,
     "start_time": "2021-11-25T16:52:37.805914",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "imputing missing values for: AMT_ANNUITY\n",
      "imputing missing values for: AMT_GOODS_PRICE\n",
      "imputing missing values for: OWN_CAR_AGE\n",
      "imputing missing values for: CNT_FAM_MEMBERS\n",
      "imputing missing values for: EXT_SOURCE_1\n",
      "imputing missing values for: EXT_SOURCE_2\n",
      "imputing missing values for: EXT_SOURCE_3\n",
      "imputing missing values for: APARTMENTS_AVG\n",
      "imputing missing values for: BASEMENTAREA_AVG\n",
      "imputing missing values for: YEARS_BEGINEXPLUATATION_AVG\n",
      "imputing missing values for: YEARS_BUILD_AVG\n",
      "imputing missing values for: COMMONAREA_AVG\n",
      "imputing missing values for: ELEVATORS_AVG\n",
      "imputing missing values for: ENTRANCES_AVG\n",
      "imputing missing values for: FLOORSMAX_AVG\n",
      "imputing missing values for: FLOORSMIN_AVG\n",
      "imputing missing values for: LANDAREA_AVG\n",
      "imputing missing values for: LIVINGAPARTMENTS_AVG\n",
      "imputing missing values for: LIVINGAREA_AVG\n",
      "imputing missing values for: NONLIVINGAPARTMENTS_AVG\n",
      "imputing missing values for: NONLIVINGAREA_AVG\n",
      "imputing missing values for: APARTMENTS_MODE\n",
      "imputing missing values for: BASEMENTAREA_MODE\n",
      "imputing missing values for: YEARS_BEGINEXPLUATATION_MODE\n",
      "imputing missing values for: YEARS_BUILD_MODE\n",
      "imputing missing values for: COMMONAREA_MODE\n",
      "imputing missing values for: ELEVATORS_MODE\n",
      "imputing missing values for: ENTRANCES_MODE\n",
      "imputing missing values for: FLOORSMAX_MODE\n",
      "imputing missing values for: FLOORSMIN_MODE\n",
      "imputing missing values for: LANDAREA_MODE\n",
      "imputing missing values for: LIVINGAPARTMENTS_MODE\n",
      "imputing missing values for: LIVINGAREA_MODE\n",
      "imputing missing values for: NONLIVINGAPARTMENTS_MODE\n",
      "imputing missing values for: NONLIVINGAREA_MODE\n",
      "imputing missing values for: APARTMENTS_MEDI\n",
      "imputing missing values for: BASEMENTAREA_MEDI\n",
      "imputing missing values for: YEARS_BEGINEXPLUATATION_MEDI\n",
      "imputing missing values for: YEARS_BUILD_MEDI\n",
      "imputing missing values for: COMMONAREA_MEDI\n",
      "imputing missing values for: ELEVATORS_MEDI\n",
      "imputing missing values for: ENTRANCES_MEDI\n",
      "imputing missing values for: FLOORSMAX_MEDI\n",
      "imputing missing values for: FLOORSMIN_MEDI\n",
      "imputing missing values for: LANDAREA_MEDI\n",
      "imputing missing values for: LIVINGAPARTMENTS_MEDI\n",
      "imputing missing values for: LIVINGAREA_MEDI\n",
      "imputing missing values for: NONLIVINGAPARTMENTS_MEDI\n",
      "imputing missing values for: NONLIVINGAREA_MEDI\n",
      "imputing missing values for: TOTALAREA_MODE\n",
      "imputing missing values for: OBS_30_CNT_SOCIAL_CIRCLE\n",
      "imputing missing values for: DEF_30_CNT_SOCIAL_CIRCLE\n",
      "imputing missing values for: OBS_60_CNT_SOCIAL_CIRCLE\n",
      "imputing missing values for: DEF_60_CNT_SOCIAL_CIRCLE\n",
      "imputing missing values for: DAYS_LAST_PHONE_CHANGE\n",
      "imputing missing values for: AMT_REQ_CREDIT_BUREAU_HOUR\n",
      "imputing missing values for: AMT_REQ_CREDIT_BUREAU_DAY\n",
      "imputing missing values for: AMT_REQ_CREDIT_BUREAU_WEEK\n",
      "imputing missing values for: AMT_REQ_CREDIT_BUREAU_MON\n",
      "imputing missing values for: AMT_REQ_CREDIT_BUREAU_QRT\n",
      "imputing missing values for: AMT_REQ_CREDIT_BUREAU_YEAR\n"
     ]
    }
   ],
   "source": [
    "# impute the missing values and create the missing value indicator variables for each numeric column.\n",
    "df_numeric = df.select_dtypes(include=[np.number])\n",
    "numeric_cols = df_numeric.columns.values\n",
    "\n",
    "for col in numeric_cols:\n",
    "    missing = df[col].isnull()\n",
    "    num_missing = np.sum(missing)\n",
    "    \n",
    "    if num_missing > 0:  # only do the imputation for the columns that have missing values.\n",
    "        print('imputing missing values for: {}'.format(col))\n",
    "        df['{}_ismissing'.format(col)] = missing\n",
    "        # med = df[col].median()\n",
    "        # df[col] = df[col].fillna(med)\n",
    "        df[col] = df[col].fillna(-999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "civil-square",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:38.136011Z",
     "iopub.status.busy": "2021-11-25T16:52:38.127016Z",
     "iopub.status.idle": "2021-11-25T16:52:38.271014Z",
     "shell.execute_reply": "2021-11-25T16:52:38.272011Z"
    },
    "papermill": {
     "duration": 0.215001,
     "end_time": "2021-11-25T16:52:38.272011",
     "exception": false,
     "start_time": "2021-11-25T16:52:38.057010",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "imputing missing values for: NAME_TYPE_SUITE\n",
      "imputing missing values for: OCCUPATION_TYPE\n",
      "imputing missing values for: FONDKAPREMONT_MODE\n",
      "imputing missing values for: HOUSETYPE_MODE\n",
      "imputing missing values for: WALLSMATERIAL_MODE\n",
      "imputing missing values for: EMERGENCYSTATE_MODE\n"
     ]
    }
   ],
   "source": [
    "# impute the missing values and create the missing value indicator variables for each non-numeric column.\n",
    "df_non_numeric = df.select_dtypes(exclude=[np.number])\n",
    "non_numeric_cols = df_non_numeric.columns.values\n",
    "\n",
    "for col in non_numeric_cols:\n",
    "    missing = df[col].isnull()\n",
    "    num_missing = np.sum(missing)\n",
    "    \n",
    "    if num_missing > 0:  # only do the imputation for the columns that have missing values.\n",
    "        print('imputing missing values for: {}'.format(col))\n",
    "        df['{}_ismissing'.format(col)] = missing\n",
    "        \n",
    "        # top = df[col].describe()['top'] # impute with the most frequent value.\n",
    "        # df[col] = df[col].fillna(top)\n",
    "        df[col] = df[col].fillna('missing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "complex-chuck",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:38.344048Z",
     "iopub.status.busy": "2021-11-25T16:52:38.343049Z",
     "iopub.status.idle": "2021-11-25T16:52:38.373014Z",
     "shell.execute_reply": "2021-11-25T16:52:38.374011Z"
    },
    "papermill": {
     "duration": 0.069961,
     "end_time": "2021-11-25T16:52:38.375012",
     "exception": false,
     "start_time": "2021-11-25T16:52:38.305051",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SK_ID_CURR\n",
      "TARGET\n",
      "NAME_CONTRACT_TYPE\n",
      "CODE_GENDER\n",
      "FLAG_OWN_CAR"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "FLAG_OWN_REALTY\n",
      "CNT_CHILDREN\n",
      "AMT_INCOME_TOTAL\n",
      "AMT_CREDIT\n",
      "AMT_ANNUITY\n",
      "AMT_GOODS_PRICE\n",
      "NAME_TYPE_SUITE\n",
      "NAME_INCOME_TYPE\n",
      "NAME_EDUCATION_TYPE\n",
      "NAME_FAMILY_STATUS\n",
      "NAME_HOUSING_TYPE\n",
      "REGION_POPULATION_RELATIVE\n",
      "DAYS_BIRTH\n",
      "DAYS_EMPLOYED\n",
      "DAYS_REGISTRATION\n",
      "DAYS_ID_PUBLISH\n",
      "OWN_CAR_AGE\n",
      "FLAG_MOBIL\n",
      "FLAG_EMP_PHONE\n",
      "FLAG_WORK_PHONE\n",
      "FLAG_CONT_MOBILE\n",
      "FLAG_PHONE\n",
      "FLAG_EMAIL\n",
      "OCCUPATION_TYPE\n",
      "CNT_FAM_MEMBERS\n",
      "REGION_RATING_CLIENT\n",
      "REGION_RATING_CLIENT_W_CITY\n",
      "WEEKDAY_APPR_PROCESS_START\n",
      "HOUR_APPR_PROCESS_START\n",
      "REG_REGION_NOT_LIVE_REGION\n",
      "REG_REGION_NOT_WORK_REGION\n",
      "LIVE_REGION_NOT_WORK_REGION\n",
      "REG_CITY_NOT_LIVE_CITY\n",
      "REG_CITY_NOT_WORK_CITY\n",
      "LIVE_CITY_NOT_WORK_CITY\n",
      "ORGANIZATION_TYPE\n",
      "EXT_SOURCE_1\n",
      "EXT_SOURCE_2\n",
      "EXT_SOURCE_3\n",
      "APARTMENTS_AVG\n",
      "BASEMENTAREA_AVG\n",
      "YEARS_BEGINEXPLUATATION_AVG\n",
      "YEARS_BUILD_AVG\n",
      "COMMONAREA_AVG\n",
      "ELEVATORS_AVG\n",
      "ENTRANCES_AVG\n",
      "FLOORSMAX_AVG\n",
      "FLOORSMIN_AVG\n",
      "LANDAREA_AVG\n",
      "LIVINGAPARTMENTS_AVG\n",
      "LIVINGAREA_AVG\n",
      "NONLIVINGAPARTMENTS_AVG\n",
      "NONLIVINGAREA_AVG\n",
      "APARTMENTS_MODE\n",
      "BASEMENTAREA_MODE\n",
      "YEARS_BEGINEXPLUATATION_MODE\n",
      "YEARS_BUILD_MODE\n",
      "COMMONAREA_MODE\n",
      "ELEVATORS_MODE\n",
      "ENTRANCES_MODE\n",
      "FLOORSMAX_MODE\n",
      "FLOORSMIN_MODE\n",
      "LANDAREA_MODE\n",
      "LIVINGAPARTMENTS_MODE\n",
      "LIVINGAREA_MODE\n",
      "NONLIVINGAPARTMENTS_MODE\n",
      "NONLIVINGAREA_MODE\n",
      "APARTMENTS_MEDI\n",
      "BASEMENTAREA_MEDI\n",
      "YEARS_BEGINEXPLUATATION_MEDI\n",
      "YEARS_BUILD_MEDI\n",
      "COMMONAREA_MEDI\n",
      "ELEVATORS_MEDI\n",
      "ENTRANCES_MEDI\n",
      "FLOORSMAX_MEDI\n",
      "FLOORSMIN_MEDI\n",
      "LANDAREA_MEDI\n",
      "LIVINGAPARTMENTS_MEDI\n",
      "LIVINGAREA_MEDI\n",
      "NONLIVINGAPARTMENTS_MEDI\n",
      "NONLIVINGAREA_MEDI\n",
      "FONDKAPREMONT_MODE\n",
      "HOUSETYPE_MODE\n",
      "TOTALAREA_MODE\n",
      "WALLSMATERIAL_MODE\n",
      "EMERGENCYSTATE_MODE\n",
      "OBS_30_CNT_SOCIAL_CIRCLE\n",
      "DEF_30_CNT_SOCIAL_CIRCLE\n",
      "OBS_60_CNT_SOCIAL_CIRCLE\n",
      "DEF_60_CNT_SOCIAL_CIRCLE\n",
      "DAYS_LAST_PHONE_CHANGE\n",
      "FLAG_DOCUMENT_2\n",
      "FLAG_DOCUMENT_3\n",
      "FLAG_DOCUMENT_4\n",
      "FLAG_DOCUMENT_5\n",
      "FLAG_DOCUMENT_6\n",
      "FLAG_DOCUMENT_7\n",
      "FLAG_DOCUMENT_8\n",
      "FLAG_DOCUMENT_9\n",
      "FLAG_DOCUMENT_10\n",
      "FLAG_DOCUMENT_11\n",
      "FLAG_DOCUMENT_12\n",
      "FLAG_DOCUMENT_13\n",
      "FLAG_DOCUMENT_14\n",
      "FLAG_DOCUMENT_15\n",
      "FLAG_DOCUMENT_16\n",
      "FLAG_DOCUMENT_17\n",
      "FLAG_DOCUMENT_18\n",
      "FLAG_DOCUMENT_19\n",
      "FLAG_DOCUMENT_20\n",
      "FLAG_DOCUMENT_21\n",
      "AMT_REQ_CREDIT_BUREAU_HOUR\n",
      "AMT_REQ_CREDIT_BUREAU_DAY\n",
      "AMT_REQ_CREDIT_BUREAU_WEEK\n",
      "AMT_REQ_CREDIT_BUREAU_MON\n",
      "AMT_REQ_CREDIT_BUREAU_QRT\n",
      "AMT_REQ_CREDIT_BUREAU_YEAR\n",
      "AMT_ANNUITY_ismissing\n",
      "AMT_GOODS_PRICE_ismissing\n",
      "NAME_TYPE_SUITE_ismissing\n",
      "OWN_CAR_AGE_ismissing\n",
      "OCCUPATION_TYPE_ismissing\n",
      "CNT_FAM_MEMBERS_ismissing\n",
      "EXT_SOURCE_1_ismissing\n",
      "EXT_SOURCE_2_ismissing\n",
      "EXT_SOURCE_3_ismissing\n",
      "APARTMENTS_AVG_ismissing\n",
      "BASEMENTAREA_AVG_ismissing\n",
      "YEARS_BEGINEXPLUATATION_AVG_ismissing\n",
      "YEARS_BUILD_AVG_ismissing\n",
      "COMMONAREA_AVG_ismissing\n",
      "ELEVATORS_AVG_ismissing\n",
      "ENTRANCES_AVG_ismissing\n",
      "FLOORSMAX_AVG_ismissing\n",
      "FLOORSMIN_AVG_ismissing\n",
      "LANDAREA_AVG_ismissing\n",
      "LIVINGAPARTMENTS_AVG_ismissing\n",
      "LIVINGAREA_AVG_ismissing\n",
      "NONLIVINGAPARTMENTS_AVG_ismissing\n",
      "NONLIVINGAREA_AVG_ismissing\n",
      "APARTMENTS_MODE_ismissing\n",
      "BASEMENTAREA_MODE_ismissing\n",
      "YEARS_BEGINEXPLUATATION_MODE_ismissing\n",
      "YEARS_BUILD_MODE_ismissing\n",
      "COMMONAREA_MODE_ismissing\n",
      "ELEVATORS_MODE_ismissing\n",
      "ENTRANCES_MODE_ismissing\n",
      "FLOORSMAX_MODE_ismissing\n",
      "FLOORSMIN_MODE_ismissing\n",
      "LANDAREA_MODE_ismissing\n",
      "LIVINGAPARTMENTS_MODE_ismissing\n",
      "LIVINGAREA_MODE_ismissing\n",
      "NONLIVINGAPARTMENTS_MODE_ismissing\n",
      "NONLIVINGAREA_MODE_ismissing\n",
      "APARTMENTS_MEDI_ismissing\n",
      "BASEMENTAREA_MEDI_ismissing\n",
      "YEARS_BEGINEXPLUATATION_MEDI_ismissing\n",
      "YEARS_BUILD_MEDI_ismissing\n",
      "COMMONAREA_MEDI_ismissing\n",
      "ELEVATORS_MEDI_ismissing\n",
      "ENTRANCES_MEDI_ismissing\n",
      "FLOORSMAX_MEDI_ismissing\n",
      "FLOORSMIN_MEDI_ismissing\n",
      "LANDAREA_MEDI_ismissing\n",
      "LIVINGAPARTMENTS_MEDI_ismissing\n",
      "LIVINGAREA_MEDI_ismissing\n",
      "NONLIVINGAPARTMENTS_MEDI_ismissing\n",
      "NONLIVINGAREA_MEDI_ismissing\n",
      "FONDKAPREMONT_MODE_ismissing\n",
      "HOUSETYPE_MODE_ismissing\n",
      "TOTALAREA_MODE_ismissing\n",
      "WALLSMATERIAL_MODE_ismissing\n",
      "EMERGENCYSTATE_MODE_ismissing\n",
      "OBS_30_CNT_SOCIAL_CIRCLE_ismissing\n",
      "DEF_30_CNT_SOCIAL_CIRCLE_ismissing\n",
      "OBS_60_CNT_SOCIAL_CIRCLE_ismissing\n",
      "DEF_60_CNT_SOCIAL_CIRCLE_ismissing\n",
      "DAYS_LAST_PHONE_CHANGE_ismissing\n",
      "AMT_REQ_CREDIT_BUREAU_HOUR_ismissing\n",
      "AMT_REQ_CREDIT_BUREAU_DAY_ismissing\n",
      "AMT_REQ_CREDIT_BUREAU_WEEK_ismissing\n",
      "AMT_REQ_CREDIT_BUREAU_MON_ismissing\n",
      "AMT_REQ_CREDIT_BUREAU_QRT_ismissing\n",
      "AMT_REQ_CREDIT_BUREAU_YEAR_ismissing\n",
      "num_missing\n"
     ]
    }
   ],
   "source": [
    "for i in df.columns:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "saving-controversy",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:38.475016Z",
     "iopub.status.busy": "2021-11-25T16:52:38.464010Z",
     "iopub.status.idle": "2021-11-25T16:52:38.524010Z",
     "shell.execute_reply": "2021-11-25T16:52:38.523011Z"
    },
    "papermill": {
     "duration": 0.110998,
     "end_time": "2021-11-25T16:52:38.524010",
     "exception": false,
     "start_time": "2021-11-25T16:52:38.413012",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# then drop the ismissing columns\n",
    "df.drop(list(df.filter(regex = '_ismissing')), axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "configured-delicious",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:38.637011Z",
     "iopub.status.busy": "2021-11-25T16:52:38.636012Z",
     "iopub.status.idle": "2021-11-25T16:52:38.683020Z",
     "shell.execute_reply": "2021-11-25T16:52:38.684014Z"
    },
    "papermill": {
     "duration": 0.124001,
     "end_time": "2021-11-25T16:52:38.684014",
     "exception": false,
     "start_time": "2021-11-25T16:52:38.560013",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop(df[['num_missing']], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "legendary-ecology",
   "metadata": {
    "papermill": {
     "duration": 0.037002,
     "end_time": "2021-11-25T16:52:38.765013",
     "exception": false,
     "start_time": "2021-11-25T16:52:38.728011",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# No more missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "split-landing",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:38.840011Z",
     "iopub.status.busy": "2021-11-25T16:52:38.839013Z",
     "iopub.status.idle": "2021-11-25T16:52:38.933010Z",
     "shell.execute_reply": "2021-11-25T16:52:38.934013Z"
    },
    "papermill": {
     "duration": 0.132999,
     "end_time": "2021-11-25T16:52:38.934013",
     "exception": false,
     "start_time": "2021-11-25T16:52:38.801014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().any().any()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "otherwise-jungle",
   "metadata": {
    "papermill": {
     "duration": 0.036005,
     "end_time": "2021-11-25T16:52:39.005016",
     "exception": false,
     "start_time": "2021-11-25T16:52:38.969011",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# No duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "blessed-moral",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:39.143012Z",
     "iopub.status.busy": "2021-11-25T16:52:39.142016Z",
     "iopub.status.idle": "2021-11-25T16:52:39.157013Z",
     "shell.execute_reply": "2021-11-25T16:52:39.156012Z"
    },
    "papermill": {
     "duration": 0.111999,
     "end_time": "2021-11-25T16:52:39.157013",
     "exception": false,
     "start_time": "2021-11-25T16:52:39.045014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.drop('SK_ID_CURR', inplace=True, axis=\"columns\") # Drop the ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "champion-analyst",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:39.240049Z",
     "iopub.status.busy": "2021-11-25T16:52:39.239038Z",
     "iopub.status.idle": "2021-11-25T16:52:39.254057Z",
     "shell.execute_reply": "2021-11-25T16:52:39.254057Z"
    },
    "papermill": {
     "duration": 0.058043,
     "end_time": "2021-11-25T16:52:39.255056",
     "exception": false,
     "start_time": "2021-11-25T16:52:39.197013",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# getting y\n",
    "df_y = pd.DataFrame(df.TARGET)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "macro-stack",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:39.438014Z",
     "iopub.status.busy": "2021-11-25T16:52:39.382013Z",
     "iopub.status.idle": "2021-11-25T16:52:39.679629Z",
     "shell.execute_reply": "2021-11-25T16:52:39.678593Z"
    },
    "papermill": {
     "duration": 0.38858,
     "end_time": "2021-11-25T16:52:39.679629",
     "exception": false,
     "start_time": "2021-11-25T16:52:39.291049",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# getting X\n",
    "df.drop(df[['TARGET']], axis=\"columns\", inplace=True)\n",
    "df_X = pd.get_dummies(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "optional-encyclopedia",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:39.753626Z",
     "iopub.status.busy": "2021-11-25T16:52:39.752632Z",
     "iopub.status.idle": "2021-11-25T16:52:39.756592Z",
     "shell.execute_reply": "2021-11-25T16:52:39.757629Z"
    },
    "papermill": {
     "duration": 0.043037,
     "end_time": "2021-11-25T16:52:39.757629",
     "exception": false,
     "start_time": "2021-11-25T16:52:39.714592",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " df_X :  (98491, 250) \n",
      " df_y :  (98491, 1)\n"
     ]
    }
   ],
   "source": [
    "# verifying shapes\n",
    "print(\"\\n df_X : \", df_X.shape, \"\\n df_y : \", df_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "nuclear-stress",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-25T16:52:39.831634Z",
     "iopub.status.busy": "2021-11-25T16:52:39.831634Z",
     "iopub.status.idle": "2021-11-25T16:52:51.351211Z",
     "shell.execute_reply": "2021-11-25T16:52:51.351211Z"
    },
    "papermill": {
     "duration": 11.558585,
     "end_time": "2021-11-25T16:52:51.351211",
     "exception": false,
     "start_time": "2021-11-25T16:52:39.792626",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# store x and y\n",
    "df_X.to_csv(path.join(\"2_dataprep\", \"prep_train_X.csv\"), index=False)\n",
    "df_y.to_csv(path.join(\"2_dataprep\", \"prep_train_y.csv\"), index=False)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e2451e044033ce079ff6ade6403154a5557f0feb2168df54e5a5df33c651b5ee"
  },
  "kernelspec": {
   "display_name": "Python 3.7.0 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 29.286662,
   "end_time": "2021-11-25T16:52:51.826854",
   "environment_variables": {},
   "exception": null,
   "input_path": "1_data_preparation.ipynb",
   "output_path": "1_data_preparation.ipynb",
   "parameters": {},
   "start_time": "2021-11-25T16:52:22.540192",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}