generate_schema.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6ef56721-ce11-4f3e-a1bd-8588a38fa8ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "import json\n",
    "from modelop.schema.infer import infer_schema, set_monitoring_parameters\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "36f08cba-8f9a-4771-9403-3fa48fcb3897",
   "metadata": {},
   "outputs": [],
   "source": [
    "# both baseline and sample should have the same schema\n",
    "df = pandas.read_json('df_baseline_scored.json', orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ce927cc2-c5f0-40d8-91a0-90d676846ac3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_schema = infer_schema(data=df, schema_name='input_schema.avsc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "254cc29b-53a6-43da-ad4d-f8aeecd6dcd8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>type</th>\n",
       "      <th>dataClass</th>\n",
       "      <th>role</th>\n",
       "      <th>protectedClass</th>\n",
       "      <th>driftCandidate</th>\n",
       "      <th>specialValues</th>\n",
       "      <th>scoringOptional</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Id</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>identifier</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>prediction</td>\n",
       "      <td>float</td>\n",
       "      <td>numerical</td>\n",
       "      <td>score</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SalePrice</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>eOverallQual_TotalSF</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OverallQual</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>eTotalSF</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>GrLivArea</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ExterQual</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>KitchenQual</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>GarageCars</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>eTotalBathrooms</td>\n",
       "      <td>float</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>BsmtQual</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>GarageArea</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>TotalBsmtSF</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>GarageFinish</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>YearBuilt</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>eHasGarage</td>\n",
       "      <td>boolean</td>\n",
       "      <td>categorical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>TotRmsAbvGrd</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>eHasRemodeling</td>\n",
       "      <td>boolean</td>\n",
       "      <td>categorical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>FireplaceQu</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>MasVnrArea</td>\n",
       "      <td>int</td>\n",
       "      <td>numerical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>eHasFireplace</td>\n",
       "      <td>boolean</td>\n",
       "      <td>categorical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>eHasBsmt</td>\n",
       "      <td>boolean</td>\n",
       "      <td>categorical</td>\n",
       "      <td>predictor</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    name     type    dataClass        role  protectedClass  \\\n",
       "0                     Id      int    numerical  identifier           False   \n",
       "1             prediction    float    numerical       score           False   \n",
       "2              SalePrice      int    numerical   predictor           False   \n",
       "3   eOverallQual_TotalSF      int    numerical   predictor           False   \n",
       "4            OverallQual      int    numerical   predictor           False   \n",
       "5               eTotalSF      int    numerical   predictor           False   \n",
       "6              GrLivArea      int    numerical   predictor           False   \n",
       "7              ExterQual      int    numerical   predictor           False   \n",
       "8            KitchenQual      int    numerical   predictor           False   \n",
       "9             GarageCars      int    numerical   predictor           False   \n",
       "10       eTotalBathrooms    float    numerical   predictor           False   \n",
       "11              BsmtQual      int    numerical   predictor           False   \n",
       "12            GarageArea      int    numerical   predictor           False   \n",
       "13           TotalBsmtSF      int    numerical   predictor           False   \n",
       "14          GarageFinish      int    numerical   predictor           False   \n",
       "15             YearBuilt      int    numerical   predictor           False   \n",
       "16            eHasGarage  boolean  categorical   predictor           False   \n",
       "17          TotRmsAbvGrd      int    numerical   predictor           False   \n",
       "18        eHasRemodeling  boolean  categorical   predictor           False   \n",
       "19           FireplaceQu      int    numerical   predictor           False   \n",
       "20            MasVnrArea      int    numerical   predictor           False   \n",
       "21         eHasFireplace  boolean  categorical   predictor           False   \n",
       "22              eHasBsmt  boolean  categorical   predictor           False   \n",
       "\n",
       "    driftCandidate specialValues  scoringOptional  \n",
       "0            False            []            False  \n",
       "1             True            []             True  \n",
       "2             True            []            False  \n",
       "3             True            []            False  \n",
       "4             True            []            False  \n",
       "5             True            []            False  \n",
       "6             True            []            False  \n",
       "7             True            []            False  \n",
       "8             True            []            False  \n",
       "9             True            []            False  \n",
       "10            True            []            False  \n",
       "11            True            []            False  \n",
       "12            True            []            False  \n",
       "13            True            []            False  \n",
       "14            True            []            False  \n",
       "15            True            []            False  \n",
       "16            True            []            False  \n",
       "17            True            []            False  \n",
       "18            True            []            False  \n",
       "19            True            []            False  \n",
       "20            True            []            False  \n",
       "21            True            []            False  \n",
       "22            True            []            False  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas.DataFrame(df_schema['fields'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5b1a57e8-6491-4c16-99ff-b71c604db4ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('input_schema.avsc', 'w') as fp:\n",
    "    json.dump(df_schema, fp, indent=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d414b877-857e-4eba-b62c-094fc3ee2a4e",
   "metadata": {},
   "source": [
    "As `SalePrice` is our target variable and the schema generator did not correctly specifiy it as such, we'll have to manually edit that portion of the schema."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ef548940-01ec-4691-9bcd-a55037d55552",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'predictors': ['eHasGarage',\n",
       "  'eHasRemodeling',\n",
       "  'eHasFireplace',\n",
       "  'eHasBsmt',\n",
       "  'eOverallQual_TotalSF',\n",
       "  'OverallQual',\n",
       "  'eTotalSF',\n",
       "  'GrLivArea',\n",
       "  'ExterQual',\n",
       "  'KitchenQual',\n",
       "  'GarageCars',\n",
       "  'eTotalBathrooms',\n",
       "  'BsmtQual',\n",
       "  'GarageArea',\n",
       "  'TotalBsmtSF',\n",
       "  'GarageFinish',\n",
       "  'YearBuilt',\n",
       "  'TotRmsAbvGrd',\n",
       "  'FireplaceQu',\n",
       "  'MasVnrArea'],\n",
       " 'categorical_columns': ['eHasGarage',\n",
       "  'eHasRemodeling',\n",
       "  'eHasFireplace',\n",
       "  'eHasBsmt'],\n",
       " 'numerical_columns': ['eOverallQual_TotalSF',\n",
       "  'OverallQual',\n",
       "  'eTotalSF',\n",
       "  'GrLivArea',\n",
       "  'ExterQual',\n",
       "  'KitchenQual',\n",
       "  'GarageCars',\n",
       "  'eTotalBathrooms',\n",
       "  'BsmtQual',\n",
       "  'GarageArea',\n",
       "  'TotalBsmtSF',\n",
       "  'GarageFinish',\n",
       "  'YearBuilt',\n",
       "  'TotRmsAbvGrd',\n",
       "  'FireplaceQu',\n",
       "  'MasVnrArea'],\n",
       " 'identifier_columns': ['Id'],\n",
       " 'weight_column': None,\n",
       " 'score_column': 'prediction',\n",
       " 'label_column': 'ground_truth',\n",
       " 'label_type': 'numerical',\n",
       " 'feature_dataclass': {'prediction': 'numerical',\n",
       "  'eOverallQual_TotalSF': 'numerical',\n",
       "  'OverallQual': 'numerical',\n",
       "  'eTotalSF': 'numerical',\n",
       "  'GrLivArea': 'numerical',\n",
       "  'ExterQual': 'numerical',\n",
       "  'KitchenQual': 'numerical',\n",
       "  'GarageCars': 'numerical',\n",
       "  'eTotalBathrooms': 'numerical',\n",
       "  'BsmtQual': 'numerical',\n",
       "  'GarageArea': 'numerical',\n",
       "  'TotalBsmtSF': 'numerical',\n",
       "  'GarageFinish': 'numerical',\n",
       "  'YearBuilt': 'numerical',\n",
       "  'eHasGarage': 'categorical',\n",
       "  'TotRmsAbvGrd': 'numerical',\n",
       "  'eHasRemodeling': 'categorical',\n",
       "  'FireplaceQu': 'numerical',\n",
       "  'MasVnrArea': 'numerical',\n",
       "  'eHasFireplace': 'categorical',\n",
       "  'eHasBsmt': 'categorical'},\n",
       " 'protected_classes': [],\n",
       " 'special_values': {'prediction': [],\n",
       "  'eOverallQual_TotalSF': [],\n",
       "  'OverallQual': [],\n",
       "  'eTotalSF': [],\n",
       "  'GrLivArea': [],\n",
       "  'ExterQual': [],\n",
       "  'KitchenQual': [],\n",
       "  'GarageCars': [],\n",
       "  'eTotalBathrooms': [],\n",
       "  'BsmtQual': [],\n",
       "  'GarageArea': [],\n",
       "  'TotalBsmtSF': [],\n",
       "  'GarageFinish': [],\n",
       "  'YearBuilt': [],\n",
       "  'eHasGarage': [],\n",
       "  'TotRmsAbvGrd': [],\n",
       "  'eHasRemodeling': [],\n",
       "  'FireplaceQu': [],\n",
       "  'MasVnrArea': [],\n",
       "  'eHasFireplace': [],\n",
       "  'eHasBsmt': []}}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "monitoring_parameters = set_monitoring_parameters(df_schema)\n",
    "monitoring_parameters"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}