LFS-OldData-Inprogress.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lfs.helper import *\n",
    "from lfs.lf_categorical import *\n",
    "from lfs.lf_extration import *\n",
    "from lfs.lf_numerical import *\n",
    "from lfs.lf_unusable import *\n",
    "from lfs.lf_context_specific import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading files data_for_labelling4.xlsx\n",
      "reading files data_for_labelling8.xlsx\n",
      "reading files data_for_labelling9.xlsx\n",
      "reading files data_for_labelling5.xlsx\n",
      "reading files data_for_labelling3.xlsx\n",
      "reading files data_for_labelling2.csv.xlsx\n",
      "reading files data_for_labelling6 (1) (1).xlsx\n",
      "reading files data_for_labelling1.csv.xlsx\n",
      "reading files data_for_labelling0.csv.xlsx\n",
      "reading files data_for_labelling7.xlsx\n"
     ]
    }
   ],
   "source": [
    "number_of_files = 1000\n",
    "base_dir = 'data/labels/'\n",
    "ext = '.xlsx'\n",
    "df = pd.DataFrame()\n",
    "for file_name in os.listdir(base_dir):\n",
    "    if number_of_files < 0: break\n",
    "    if not file_name.endswith(ext): continue\n",
    "    number_of_files -= 1\n",
    "    print('reading files', file_name)\n",
    "    df = df.append(pd.read_excel(base_dir + file_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['% nans', 'Attribute_name', 'Check', 'Num of nans', 'Reason', 'Record_id', 'Total_val', 'Unnamed: 2', 'Unnamed: 9', 'check', 'max_val', 'mean', 'min_val', 'num of dist_val', 'reason', 'sample_1', 'sample_2', 'sample_3', 'sample_4', 'sample_5', 'std_dev', 'y_Arun', 'y_act', 'y_pred']\n"
     ]
    }
   ],
   "source": [
    "cols = df.columns.tolist()\n",
    "print(cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['% nans'] = df['Num of nans'] / df['Total_val']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['y_act','Reason','Attribute_name','Total_val','num of dist_val','% nans'] + cols[10:13] + ['std_dev'] + cols[15:-4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>StratificationCategory2</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      y_act Reason           Attribute_name  Total_val  num of dist_val  \\\n",
       "0  Unusable      l  StratificationCategory2     403984                1   \n",
       "\n",
       "     % nans  max_val  mean  min_val  std_dev sample_1 sample_2 sample_3  \\\n",
       "0  0.803648      0.0   0.0      0.0      0.0      NaN      NaN      NaN   \n",
       "\n",
       "  sample_4 sample_5  \n",
       "0      NaN      NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.reindex(columns=cols)\n",
    "df[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['y_act', 'Reason', 'Attribute_name', 'Total_val', 'num of dist_val', '% nans', 'max_val', 'mean', 'min_val', 'std_dev', 'sample_1', 'sample_2', 'sample_3', 'sample_4', 'sample_5']\n"
     ]
    }
   ],
   "source": [
    "print(df.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df.columns.tolist()[-5:]] = df[df.columns.tolist()[-5:]].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Unusable', 'Usable with extraction',\n",
       "       'Usable directly categorical', 'Context_specific',\n",
       "       'Usable directly numeric', 'Usable Directly Numeric',\n",
       "       'Usable Directly Categorical', 'Usable with Extraction',\n",
       "       'Usable Directly Categorical ', 'Usable with Extration',\n",
       "       'Usable With Extraction', 'Usable with Extraction ', 'unusable',\n",
       "       'Usable Directly numeric', 'Usable directly Categorical',\n",
       "       'Usable Directly categorical'], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.y_act.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['y_act'] = df.apply(lambda row: label_mapper(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = {'Usable Directly Categorical':    1, \n",
    "              'Usable Directly Numeric':        2, \n",
    "              'Usable With Extraction':         3, \n",
    "              'Unusable':                       4, \n",
    "              'Context_Specific':               5, \n",
    "              'None':                           0\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Context_Specific': 5,\n",
       " 'None': 0,\n",
       " 'Unusable': 4,\n",
       " 'Usable Directly Categorical': 1,\n",
       " 'Usable Directly Numeric': 2,\n",
       " 'Usable With Extraction': 3}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "y_act\n",
       "Context_Specific               2050\n",
       "Unusable                        891\n",
       "Usable Directly Categorical    2087\n",
       "Usable Directly Numeric        5063\n",
       "Usable With Extraction          650\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(df.y_act).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories_size = {'Context_Specific':              2050,\n",
    "                   'Unusable':                      891,\n",
    "                   'Usable Directly Categorical':   2087,\n",
    "                   'Usable Directly Numeric':       5063,\n",
    "                   'Usable With Extraction':        650\n",
    "                   }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "catelens = [0, 2087, 5063, 650, 891, 2050]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 7, 2, 17, 2, 0]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_key_word('location', df, 'Attribute_name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 19, 2, 9, 10, 11]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_key_word('group', df, 'Attribute_name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 19, 21, 1, 8, 7]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_key_word('lot', df, 'Attribute_name')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Usable With Extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|Total|Matched|Mismatched| Abstained| Accuracy | Coverage|\n",
    "| --- | --- | --- | --- | --- | --- |\n",
    "|   650  |  581 | 559  | 69| 58.69 | 89.85 |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lfs_ext = [lf_extractable_units, lf_extractable_number_sci, lf_extractable_pattern, lf_date_extraction_name, lf_extractable_name, lf_extractable_list, lf_extractable_sample_length, lf_date_extraction_samples, lf_extractable_email_url, lf_same_len_string]\n",
    "len(lfs_ext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labeling function names       \t       nan  category   numeric   extract  unusable   context\n",
      "lf_extractable_units          \t         0         1         0         6         0         2\n",
      "lf_extractable_number_sci     \t         0         1         1         0         0         0\n",
      "lf_extractable_pattern        \t         0        25         1        78        14        14\n",
      "lf_date_extraction_name       \t         0         6        21       149        12         1\n",
      "lf_extractable_name           \t         0        58        60       186        37         8\n",
      "lf_extractable_list           \t         0         2         3        27         5         0\n",
      "lf_extractable_sample_length  \t         0       123         3       274        37        57\n",
      "lf_date_extraction_samples    \t         0        19         0       264        17        51\n",
      "lf_extractable_email_url      \t         0         0         0        33        10         4\n"
     ]
    }
   ],
   "source": [
    "cms = test_lfs(lfs_ext, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0, 227, 86, 582, 136, 129], 68, 578, 0.568359375, 0.8953846153846153)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm, abstained, mismatched, accuracy, coverage = test_lfs_onecategory(lfs_ext, df, 'Usable With Extraction')\n",
    "cm, len(abstained), len(mismatched), accuracy, coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 2087, 5063, 650, 891, 2050]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "catelens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>f</td>\n",
       "      <td>item</td>\n",
       "      <td>188939.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.001493</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Acquisitions less disposals of valuables</td>\n",
       "      <td>nan</td>\n",
       "      <td>Changes in inventories</td>\n",
       "      <td>Collective consumption expenditure</td>\n",
       "      <td>Equals: GROSS DOMESTIC PRODUCT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>h</td>\n",
       "      <td>IMF data dissemination standard</td>\n",
       "      <td>263.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.304183</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>nan</td>\n",
       "      <td>Enhanced General Data Dissemination System (e-...</td>\n",
       "      <td>Special Data Dissemination Standard (SDDS)</td>\n",
       "      <td>Special Data Dissemination Standard Plus (SDDS+)</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>f</td>\n",
       "      <td>category</td>\n",
       "      <td>188939.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>acquisitions_less_disposals_of_valuables</td>\n",
       "      <td>changes_in_inventories</td>\n",
       "      <td>collective_consumption_expenditure</td>\n",
       "      <td>equals_gross_domestic_product</td>\n",
       "      <td>exports_of_goods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>841</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>a</td>\n",
       "      <td>contrib_other</td>\n",
       "      <td>2720.0</td>\n",
       "      <td>634.0</td>\n",
       "      <td>0.691176</td>\n",
       "      <td>4330373.948</td>\n",
       "      <td>34795.657690</td>\n",
       "      <td>-0.149</td>\n",
       "      <td>194261.929000</td>\n",
       "      <td>1273</td>\n",
       "      <td>2091</td>\n",
       "      <td>2521</td>\n",
       "      <td>1575</td>\n",
       "      <td>2185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>h</td>\n",
       "      <td>label</td>\n",
       "      <td>9400.0</td>\n",
       "      <td>2235.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Finger_snapping Snare_drum Telephone</td>\n",
       "      <td>Flute Oboe Clarinet</td>\n",
       "      <td>Bass_drum Snare_drum Gunshot_or_gunfire</td>\n",
       "      <td>Bass_drum Snare_drum Electric_piano</td>\n",
       "      <td>Oboe Flute Clarinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>a</td>\n",
       "      <td>Review_Total_Positive_Word_Counts</td>\n",
       "      <td>515738.0</td>\n",
       "      <td>365.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>395.000</td>\n",
       "      <td>17.776458</td>\n",
       "      <td>0.000</td>\n",
       "      <td>21.804164</td>\n",
       "      <td>11</td>\n",
       "      <td>105</td>\n",
       "      <td>21</td>\n",
       "      <td>26</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>713</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>f</td>\n",
       "      <td>s1p2c3plant</td>\n",
       "      <td>9597.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>0.489632</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Jun</td>\n",
       "      <td>nan</td>\n",
       "      <td>2002-08-01 00:00:00</td>\n",
       "      <td>Jul</td>\n",
       "      <td>Aug</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>h</td>\n",
       "      <td>label</td>\n",
       "      <td>9400.0</td>\n",
       "      <td>2398.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Finger_snapping Scissors Shatter</td>\n",
       "      <td>Oboe Flute Clarinet</td>\n",
       "      <td>Bass_drum Snare_drum Hi-hat</td>\n",
       "      <td>Bass_drum Electric_piano Snare_drum</td>\n",
       "      <td>Oboe Flute Saxophone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>q21CoreCompCodeReview</td>\n",
       "      <td>25090.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.200638</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>nan</td>\n",
       "      <td>Code Review Capability</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>h</td>\n",
       "      <td>contentType</td>\n",
       "      <td>3122.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>HTML</td>\n",
       "      <td>RICH</td>\n",
       "      <td>VIDEO</td>\n",
       "      <td>HTML</td>\n",
       "      <td>HTML</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           y_act Reason                     Attribute_name  \\\n",
       "64   Usable Directly Categorical      f                               item   \n",
       "411  Usable Directly Categorical      h    IMF data dissemination standard   \n",
       "73   Usable Directly Categorical      f                           category   \n",
       "841      Usable Directly Numeric      a                      contrib_other   \n",
       "171  Usable Directly Categorical      h                              label   \n",
       "208      Usable Directly Numeric      a  Review_Total_Positive_Word_Counts   \n",
       "713             Context_Specific      f                        s1p2c3plant   \n",
       "166  Usable Directly Categorical      h                              label   \n",
       "438                     Unusable      l              q21CoreCompCodeReview   \n",
       "592  Usable Directly Categorical      h                        contentType   \n",
       "\n",
       "     Total_val  num of dist_val    % nans      max_val          mean  min_val  \\\n",
       "64    188939.0             18.0  0.001493        0.000      0.000000    0.000   \n",
       "411      263.0              3.0  0.304183        0.000      0.000000    0.000   \n",
       "73    188939.0             18.0  0.000000        0.000      0.000000    0.000   \n",
       "841     2720.0            634.0  0.691176  4330373.948  34795.657690   -0.149   \n",
       "171     9400.0           2235.0  0.000000        0.000      0.000000    0.000   \n",
       "208   515738.0            365.0  0.000000      395.000     17.776458    0.000   \n",
       "713     9597.0            118.0  0.489632        0.000      0.000000    0.000   \n",
       "166     9400.0           2398.0  0.000000        0.000      0.000000    0.000   \n",
       "438    25090.0              2.0  0.200638        0.000      0.000000    0.000   \n",
       "592     3122.0              3.0  0.000000        0.000      0.000000    0.000   \n",
       "\n",
       "           std_dev                                  sample_1  \\\n",
       "64        0.000000  Acquisitions less disposals of valuables   \n",
       "411       0.000000                                       nan   \n",
       "73        0.000000  acquisitions_less_disposals_of_valuables   \n",
       "841  194261.929000                                      1273   \n",
       "171       0.000000      Finger_snapping Snare_drum Telephone   \n",
       "208      21.804164                                        11   \n",
       "713       0.000000                                       Jun   \n",
       "166       0.000000          Finger_snapping Scissors Shatter   \n",
       "438       0.000000                                       nan   \n",
       "592       0.000000                                      HTML   \n",
       "\n",
       "                                              sample_2  \\\n",
       "64                                                 nan   \n",
       "411  Enhanced General Data Dissemination System (e-...   \n",
       "73                              changes_in_inventories   \n",
       "841                                               2091   \n",
       "171                                Flute Oboe Clarinet   \n",
       "208                                                105   \n",
       "713                                                nan   \n",
       "166                                Oboe Flute Clarinet   \n",
       "438                             Code Review Capability   \n",
       "592                                               RICH   \n",
       "\n",
       "                                       sample_3  \\\n",
       "64                       Changes in inventories   \n",
       "411  Special Data Dissemination Standard (SDDS)   \n",
       "73           collective_consumption_expenditure   \n",
       "841                                        2521   \n",
       "171     Bass_drum Snare_drum Gunshot_or_gunfire   \n",
       "208                                          21   \n",
       "713                         2002-08-01 00:00:00   \n",
       "166                 Bass_drum Snare_drum Hi-hat   \n",
       "438                                         nan   \n",
       "592                                       VIDEO   \n",
       "\n",
       "                                             sample_4  \\\n",
       "64                 Collective consumption expenditure   \n",
       "411  Special Data Dissemination Standard Plus (SDDS+)   \n",
       "73                      equals_gross_domestic_product   \n",
       "841                                              1575   \n",
       "171               Bass_drum Snare_drum Electric_piano   \n",
       "208                                                26   \n",
       "713                                               Jul   \n",
       "166               Bass_drum Electric_piano Snare_drum   \n",
       "438                                               nan   \n",
       "592                                              HTML   \n",
       "\n",
       "                           sample_5  \n",
       "64   Equals: GROSS DOMESTIC PRODUCT  \n",
       "411                             nan  \n",
       "73                 exports_of_goods  \n",
       "841                            2185  \n",
       "171             Oboe Flute Clarinet  \n",
       "208                               8  \n",
       "713                             Aug  \n",
       "166            Oboe Flute Saxophone  \n",
       "438                             nan  \n",
       "592                            HTML  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mismatched = pd.DataFrame(mismatched, columns=cols)\n",
    "mismatched.to_csv('results/extractable_mismatched.csv')\n",
    "mismatched.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>n</td>\n",
       "      <td>Station</td>\n",
       "      <td>513373.0</td>\n",
       "      <td>13431.0</td>\n",
       "      <td>0.506032</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>nan</td>\n",
       "      <td>D2-EX1504L3-05</td>\n",
       "      <td>D2-EX1504L3-07</td>\n",
       "      <td>19</td>\n",
       "      <td>201109281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>748</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>c</td>\n",
       "      <td>ADM1SALBNA</td>\n",
       "      <td>4861.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.376671</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>nan</td>\n",
       "      <td>National Capital Region</td>\n",
       "      <td>Cordillera Admin Region</td>\n",
       "      <td>Region I - Ilocos Region</td>\n",
       "      <td>Region II - Cagayan Valle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>h</td>\n",
       "      <td>currency</td>\n",
       "      <td>188939.0</td>\n",
       "      <td>378.0</td>\n",
       "      <td>0.001493</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>Euro</td>\n",
       "      <td>1999 ATS euro / euro</td>\n",
       "      <td>1999 ATS euro / euro</td>\n",
       "      <td>Austrian schilling</td>\n",
       "      <td>1999 BEF euro / euro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1064</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>c</td>\n",
       "      <td>topic_name</td>\n",
       "      <td>2509.0</td>\n",
       "      <td>2509.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>Sports Fan</td>\n",
       "      <td>Latin Music</td>\n",
       "      <td>English as a Second Language</td>\n",
       "      <td>Spanish Language</td>\n",
       "      <td>Italian Language</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>STREET</td>\n",
       "      <td>619129.0</td>\n",
       "      <td>389829.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>570 SOUTH FOURTH STREET NO 100</td>\n",
       "      <td>1706 GREENLEAF DR</td>\n",
       "      <td>LOCAL</td>\n",
       "      <td>15623 S CENTERLINE RD</td>\n",
       "      <td>201 E PATRICK ST UNIT 249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>INTRON</td>\n",
       "      <td>65188.0</td>\n",
       "      <td>1929.0</td>\n",
       "      <td>0.864960</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>nan</td>\n",
       "      <td>Jun-38</td>\n",
       "      <td>16/38</td>\n",
       "      <td>20/38</td>\n",
       "      <td>25/38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>value</td>\n",
       "      <td>72485.0</td>\n",
       "      <td>72061.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.962430e+15</td>\n",
       "      <td>7.871720e+12</td>\n",
       "      <td>-152522177.6</td>\n",
       "      <td>8.707530e+13</td>\n",
       "      <td>131880000000</td>\n",
       "      <td>158614000000</td>\n",
       "      <td>152894000000</td>\n",
       "      <td>141212000000</td>\n",
       "      <td>136766000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>852</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ICO</td>\n",
       "      <td>240579.0</td>\n",
       "      <td>138846.0</td>\n",
       "      <td>0.340233</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>nan</td>\n",
       "      <td>% ALBERTO MOLINA MARTINEZ</td>\n",
       "      <td>% JOANNE SOUSA</td>\n",
       "      <td>% DAVID WALRATH</td>\n",
       "      <td>% BRENDAN BURKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>c</td>\n",
       "      <td>National accounts base year</td>\n",
       "      <td>263.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>0.220532</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2000</td>\n",
       "      <td>2002-03-01 00:00:00</td>\n",
       "      <td>2002</td>\n",
       "      <td>Original chained constant price data are resca...</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>780</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>site_info</td>\n",
       "      <td>188462.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>Sidewalk: Curb side : Cutout</td>\n",
       "      <td>:</td>\n",
       "      <td>Sidewalk: Property side : Yard</td>\n",
       "      <td>Sidewalk: Curb side : Yard</td>\n",
       "      <td>Median : Cutout</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       y_act Reason               Attribute_name  Total_val  \\\n",
       "383   Usable With Extraction      n                      Station   513373.0   \n",
       "748   Usable With Extraction      c                   ADM1SALBNA     4861.0   \n",
       "68    Usable With Extraction      h                     currency   188939.0   \n",
       "1064  Usable With Extraction      c                   topic_name     2509.0   \n",
       "881   Usable With Extraction    NaN                       STREET   619129.0   \n",
       "1022  Usable With Extraction    NaN                       INTRON    65188.0   \n",
       "34    Usable With Extraction    NaN                        value    72485.0   \n",
       "852   Usable With Extraction    NaN                          ICO   240579.0   \n",
       "399   Usable With Extraction      c  National accounts base year      263.0   \n",
       "780   Usable With Extraction    NaN                    site_info   188462.0   \n",
       "\n",
       "      num of dist_val    % nans       max_val          mean      min_val  \\\n",
       "383           13431.0  0.506032  0.000000e+00  0.000000e+00          0.0   \n",
       "748              17.0  0.376671  0.000000e+00  0.000000e+00          0.0   \n",
       "68              378.0  0.001493  0.000000e+00  0.000000e+00          0.0   \n",
       "1064           2509.0  0.000000  0.000000e+00  0.000000e+00          0.0   \n",
       "881          389829.0  0.000002  0.000000e+00  0.000000e+00          0.0   \n",
       "1022           1929.0  0.864960  0.000000e+00  0.000000e+00          0.0   \n",
       "34            72061.0  0.000000  4.962430e+15  7.871720e+12 -152522177.6   \n",
       "852          138846.0  0.340233  0.000000e+00  0.000000e+00          0.0   \n",
       "399              39.0  0.220532  0.000000e+00  0.000000e+00          0.0   \n",
       "780              31.0  0.000000  0.000000e+00  0.000000e+00          0.0   \n",
       "\n",
       "           std_dev                        sample_1                   sample_2  \\\n",
       "383   0.000000e+00                             nan             D2-EX1504L3-05   \n",
       "748   0.000000e+00                             nan    National Capital Region   \n",
       "68    0.000000e+00                            Euro       1999 ATS euro / euro   \n",
       "1064  0.000000e+00                      Sports Fan                Latin Music   \n",
       "881   0.000000e+00  570 SOUTH FOURTH STREET NO 100          1706 GREENLEAF DR   \n",
       "1022  0.000000e+00                             nan                     Jun-38   \n",
       "34    8.707530e+13                    131880000000               158614000000   \n",
       "852   0.000000e+00                             nan  % ALBERTO MOLINA MARTINEZ   \n",
       "399   0.000000e+00                            2000        2002-03-01 00:00:00   \n",
       "780   0.000000e+00    Sidewalk: Curb side : Cutout                          :   \n",
       "\n",
       "                            sample_3  \\\n",
       "383                   D2-EX1504L3-07   \n",
       "748          Cordillera Admin Region   \n",
       "68              1999 ATS euro / euro   \n",
       "1064    English as a Second Language   \n",
       "881                            LOCAL   \n",
       "1022                           16/38   \n",
       "34                      152894000000   \n",
       "852                   % JOANNE SOUSA   \n",
       "399                             2002   \n",
       "780   Sidewalk: Property side : Yard   \n",
       "\n",
       "                                               sample_4  \\\n",
       "383                                                  19   \n",
       "748                            Region I - Ilocos Region   \n",
       "68                                   Austrian schilling   \n",
       "1064                                   Spanish Language   \n",
       "881                               15623 S CENTERLINE RD   \n",
       "1022                                              20/38   \n",
       "34                                         141212000000   \n",
       "852                                     % DAVID WALRATH   \n",
       "399   Original chained constant price data are resca...   \n",
       "780                          Sidewalk: Curb side : Yard   \n",
       "\n",
       "                       sample_5  \n",
       "383                   201109281  \n",
       "748   Region II - Cagayan Valle  \n",
       "68         1999 BEF euro / euro  \n",
       "1064           Italian Language  \n",
       "881   201 E PATRICK ST UNIT 249  \n",
       "1022                      25/38  \n",
       "34                 136766000000  \n",
       "852             % BRENDAN BURKE  \n",
       "399                         nan  \n",
       "780             Median : Cutout  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abstained = pd.DataFrame(abstained, columns=cols)\n",
    "abstained.to_csv('results/extractable_abstained.csv')\n",
    "abstained.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Usable Directly Categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|Total|Matched|Mismatched| Abstained| Accuracy | Coverage |\n",
    "| --- | --- | --- | --- | --- | --- |\n",
    "|   2087  |  1803 | 1515  | 284 | 59.27    | 86.39    |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lf_same_len_num_lt(row):\n",
    "    samples = get_samples(row)\n",
    "    if lf_sample_cast_to_float(row):\n",
    "        return 0\n",
    "    sample_len = [len(sample) for sample in samples]\n",
    "    sample_len_min = sum(lens<=2 for lens in sample_len)\n",
    "    if len(set(sample_len)) < 2 and sample_len_min>3:\n",
    "        return 3\n",
    "    return 0\n",
    "\n",
    "def lf_same_len_string_lt(row):\n",
    "    samples = get_samples(row)\n",
    "    if lf_cast_to_numbers(row):\n",
    "        return 0\n",
    "    sample_len = [len(sample) for sample in samples]\n",
    "    sample_len_min = sum(lens<=10 for lens in sample_len)\n",
    "    if len(set(sample_len)) < 2 and sample_len_min>3:\n",
    "        return 3\n",
    "    return 0\n",
    "\n",
    "def lf_max_eq_dist(row):\n",
    "    if row['num of dist_val'] == 1 or row['num of dist_val'] == 0:\n",
    "        return 0\n",
    "    if row['max_val'] == row['num of dist_val']:\n",
    "        return 3\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labeling function names       \t       nan  category   numeric   extract  unusable   context\n",
      "lf_same_len_num_lt            \t         0       426       228         0       155       142\n",
      "lf_same_len_string_lt         \t         0       110         0        34        61        27\n",
      "lf_max_lt_dist                \t         0       105        75         0        22       106\n"
     ]
    }
   ],
   "source": [
    "cms = test_lfs([lf_same_len_num_lt, lf_same_len_string_lt, lf_max_lt_dist], df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lfs_cate = [lf_binary_category, lf_name_category, lf_coded_abbreviation, lf_finite_set_name, lf_finite_set_sample, lf_dist_string_percentage, lf_dist_num_percentage]\n",
    "len(lfs_cate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labeling function names       \t       nan  category   numeric   extract  unusable   context\n",
      "lf_binary_category            \t         0       928        46         7       101       154\n",
      "lf_name_category              \t         0       330       100        22        48        11\n",
      "lf_coded_abbreviation         \t         0       103         0         0        27        11\n",
      "lf_finite_set_name            \t         0       396        13        63       100        27\n",
      "lf_finite_set_sample          \t         0       226         1        83        64        79\n",
      "lf_dist_string_percentage     \t         0       808       161       210       100       185\n",
      "lf_dist_num_percentage        \t         0       515       143         0        17       265\n"
     ]
    }
   ],
   "source": [
    "cms = test_lfs(lfs_cate, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0, 1803, 419, 291, 276, 529],\n",
       " 284,\n",
       " 1515,\n",
       " 0.5927021696252466,\n",
       " 0.8639195016770483)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm, abstained, mismatched, accuracy, coverage = test_lfs_onecategory(lfs_cate, df, 'Usable Directly Categorical')\n",
    "cm, len(abstained), len(mismatched), accuracy, coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 2087, 5063, 650, 891, 2050]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "catelens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>sna93_table_code</td>\n",
       "      <td>188939.0</td>\n",
       "      <td>161.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.1</td>\n",
       "      <td>Footnote</td>\n",
       "      <td>Excluding works of arts and antiques.</td>\n",
       "      <td>Includes a statistical discrepancy.</td>\n",
       "      <td>Includes Acquisitions less disposals of valuab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>581</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>a</td>\n",
       "      <td>UnitPrice</td>\n",
       "      <td>541909.0</td>\n",
       "      <td>1630.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>38970.0</td>\n",
       "      <td>4.611114</td>\n",
       "      <td>-11062.06</td>\n",
       "      <td>96.759764</td>\n",
       "      <td>2.55</td>\n",
       "      <td>3.39</td>\n",
       "      <td>2.75</td>\n",
       "      <td>7.65</td>\n",
       "      <td>4.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>job_description</td>\n",
       "      <td>22000.0</td>\n",
       "      <td>18744.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>TeamSoft is seeing an IT Support Specialist to...</td>\n",
       "      <td>The Wisconsin State Journal is seeking a flexi...</td>\n",
       "      <td>Report this job About the Job DePuy Synthes Co...</td>\n",
       "      <td>Why Join Altec? If you阵朐諍擘阵雮阵隷阵雮諍霂re consideri...</td>\n",
       "      <td>Position ID#阵朐阵擘阵雮阵隷 76162 # Positions阵朐阵擘阵雮阵隷...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>q17HirChaHardAssessSkills</td>\n",
       "      <td>25090.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.123515</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>Hard to assess skills before onsite</td>\n",
       "      <td>Hard to assess skills before onsite</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>n.status</td>\n",
       "      <td>2031.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>530</th>\n",
       "      <td>Usable With Extraction</td>\n",
       "      <td>b</td>\n",
       "      <td>value</td>\n",
       "      <td>19998.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.548805</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>nan</td>\n",
       "      <td>4 of 5 bubbles</td>\n",
       "      <td>1 of 5 bubbles</td>\n",
       "      <td>3 of 5 bubbles</td>\n",
       "      <td>5 of 5 bubbles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>820</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>farmwork3</td>\n",
       "      <td>9597.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.145462</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.245946</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.520161</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>nan</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>786</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>BoroCode</td>\n",
       "      <td>324403.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>n.company_type</td>\n",
       "      <td>59228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>o</td>\n",
       "      <td>DESCRICAO_NACIONALIDADE</td>\n",
       "      <td>21124.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>BRASILEIRA NATA</td>\n",
       "      <td>BRASILEIRA (NATURALIZADA)</td>\n",
       "      <td>PORTUGUESA COM IGUALDADE DE DIREITOS</td>\n",
       "      <td>ESTRANGEIRO</td>\n",
       "      <td>ESTRANGEIRO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       y_act Reason             Attribute_name  Total_val  \\\n",
       "62    Usable With Extraction    NaN           sna93_table_code   188939.0   \n",
       "581  Usable Directly Numeric      a                  UnitPrice   541909.0   \n",
       "338   Usable With Extraction    NaN            job_description    22000.0   \n",
       "406                 Unusable      l  q17HirChaHardAssessSkills    25090.0   \n",
       "360                 Unusable      l                   n.status     2031.0   \n",
       "530   Usable With Extraction      b                      value    19998.0   \n",
       "820         Context_Specific      a                  farmwork3     9597.0   \n",
       "786                 Unusable      l                   BoroCode   324403.0   \n",
       "289                 Unusable      l             n.company_type    59228.0   \n",
       "634         Context_Specific      o    DESCRICAO_NACIONALIDADE    21124.0   \n",
       "\n",
       "     num of dist_val    % nans  max_val      mean   min_val    std_dev  \\\n",
       "62             161.0  0.000000      0.0  0.000000      0.00   0.000000   \n",
       "581           1630.0  0.000000  38970.0  4.611114 -11062.06  96.759764   \n",
       "338          18744.0  0.000000      0.0  0.000000      0.00   0.000000   \n",
       "406              2.0  0.123515      0.0  0.000000      0.00   0.000000   \n",
       "360              0.0  1.000000      0.0  0.000000      0.00   0.000000   \n",
       "530             10.0  0.548805      0.0  0.000000      0.00   0.000000   \n",
       "820              3.0  0.145462      2.0  1.245946      0.00   0.520161   \n",
       "786              1.0  0.000000      4.0  4.000000      4.00   0.000000   \n",
       "289              0.0  1.000000      0.0  0.000000      0.00   0.000000   \n",
       "634              4.0  0.000000      0.0  0.000000      0.00   0.000000   \n",
       "\n",
       "                                              sample_1  \\\n",
       "62                                                 1.1   \n",
       "581                                               2.55   \n",
       "338  TeamSoft is seeing an IT Support Specialist to...   \n",
       "406                                                nan   \n",
       "360                                                nan   \n",
       "530                                                nan   \n",
       "820                                                  1   \n",
       "786                                                  4   \n",
       "289                                                nan   \n",
       "634                                    BRASILEIRA NATA   \n",
       "\n",
       "                                              sample_2  \\\n",
       "62                                            Footnote   \n",
       "581                                               3.39   \n",
       "338  The Wisconsin State Journal is seeking a flexi...   \n",
       "406                                                nan   \n",
       "360                                                nan   \n",
       "530                                     4 of 5 bubbles   \n",
       "820                                                  2   \n",
       "786                                                  4   \n",
       "289                                                nan   \n",
       "634                          BRASILEIRA (NATURALIZADA)   \n",
       "\n",
       "                                              sample_3  \\\n",
       "62               Excluding works of arts and antiques.   \n",
       "581                                               2.75   \n",
       "338  Report this job About the Job DePuy Synthes Co...   \n",
       "406                Hard to assess skills before onsite   \n",
       "360                                                nan   \n",
       "530                                     1 of 5 bubbles   \n",
       "820                                                nan   \n",
       "786                                                  4   \n",
       "289                                                nan   \n",
       "634               PORTUGUESA COM IGUALDADE DE DIREITOS   \n",
       "\n",
       "                                              sample_4  \\\n",
       "62                 Includes a statistical discrepancy.   \n",
       "581                                               7.65   \n",
       "338  Why Join Altec? If you阵朐諍擘阵雮阵隷阵雮諍霂re consideri...   \n",
       "406                Hard to assess skills before onsite   \n",
       "360                                                nan   \n",
       "530                                     3 of 5 bubbles   \n",
       "820                                              False   \n",
       "786                                                  4   \n",
       "289                                                nan   \n",
       "634                                        ESTRANGEIRO   \n",
       "\n",
       "                                              sample_5  \n",
       "62   Includes Acquisitions less disposals of valuab...  \n",
       "581                                               4.25  \n",
       "338  Position ID#阵朐阵擘阵雮阵隷 76162 # Positions阵朐阵擘阵雮阵隷...  \n",
       "406                                                nan  \n",
       "360                                                nan  \n",
       "530                                     5 of 5 bubbles  \n",
       "820                                                  1  \n",
       "786                                                  4  \n",
       "289                                                nan  \n",
       "634                                        ESTRANGEIRO  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mismatched = pd.DataFrame(mismatched, columns=cols)\n",
    "mismatched.to_csv('results/cate_mismatched.csv')\n",
    "mismatched.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1055</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>NaN</td>\n",
       "      <td>YEAR</td>\n",
       "      <td>1224.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>2003.500000</td>\n",
       "      <td>1992.0</td>\n",
       "      <td>6.922187</td>\n",
       "      <td>1992</td>\n",
       "      <td>1993</td>\n",
       "      <td>1994</td>\n",
       "      <td>1995</td>\n",
       "      <td>1996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>i</td>\n",
       "      <td>fastestLap</td>\n",
       "      <td>23777.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>0.773605</td>\n",
       "      <td>78.0</td>\n",
       "      <td>41.061676</td>\n",
       "      <td>2.0</td>\n",
       "      <td>17.154842</td>\n",
       "      <td>39</td>\n",
       "      <td>41</td>\n",
       "      <td>58</td>\n",
       "      <td>43</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Date</td>\n",
       "      <td>8190.0</td>\n",
       "      <td>182.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2010-05-02 00:00:00</td>\n",
       "      <td>2010-12-02 00:00:00</td>\n",
       "      <td>19/02/10</td>\n",
       "      <td>26/02/10</td>\n",
       "      <td>2010-05-03 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>j</td>\n",
       "      <td>Requisition Number</td>\n",
       "      <td>346018.0</td>\n",
       "      <td>5997.0</td>\n",
       "      <td>0.958473</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>REQ0011118</td>\n",
       "      <td>REQ0011932</td>\n",
       "      <td>REQ0011476</td>\n",
       "      <td>nan</td>\n",
       "      <td>REQ0013911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>900</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>g</td>\n",
       "      <td>Police District</td>\n",
       "      <td>64618.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>0.000573</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>590</td>\n",
       "      <td>822</td>\n",
       "      <td>830</td>\n",
       "      <td>850</td>\n",
       "      <td>871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>i</td>\n",
       "      <td>fiscal_year</td>\n",
       "      <td>68023.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.002896</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>2002.546811</td>\n",
       "      <td>1950.0</td>\n",
       "      <td>11.183397</td>\n",
       "      <td>2015</td>\n",
       "      <td>2014</td>\n",
       "      <td>2013</td>\n",
       "      <td>2012</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>885</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GROUP</td>\n",
       "      <td>619129.0</td>\n",
       "      <td>2589.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9707.0</td>\n",
       "      <td>865.599382</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1970.373963</td>\n",
       "      <td>1736</td>\n",
       "      <td>0</td>\n",
       "      <td>928</td>\n",
       "      <td>557</td>\n",
       "      <td>1961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>f</td>\n",
       "      <td>Con</td>\n",
       "      <td>1518.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>Afghanistan/Pakistan</td>\n",
       "      <td>Algeria</td>\n",
       "      <td>Angola</td>\n",
       "      <td>Angola/Burundi/Central African Republic/Democr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>g</td>\n",
       "      <td>Police District</td>\n",
       "      <td>82958.0</td>\n",
       "      <td>99.0</td>\n",
       "      <td>0.000241</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>690</td>\n",
       "      <td>590</td>\n",
       "      <td>822</td>\n",
       "      <td>642</td>\n",
       "      <td>620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>j</td>\n",
       "      <td>employer_phone</td>\n",
       "      <td>374362.0</td>\n",
       "      <td>49220.0</td>\n",
       "      <td>0.361428</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>nan</td>\n",
       "      <td>2625444811</td>\n",
       "      <td>662-254-7200</td>\n",
       "      <td>720-859-6111</td>\n",
       "      <td>206 391 7466</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            y_act Reason      Attribute_name  Total_val  \\\n",
       "1055  Usable Directly Categorical    NaN                YEAR     1224.0   \n",
       "257   Usable Directly Categorical      i          fastestLap    23777.0   \n",
       "403   Usable Directly Categorical    NaN                Date     8190.0   \n",
       "245   Usable Directly Categorical      j  Requisition Number   346018.0   \n",
       "900   Usable Directly Categorical      g     Police District    64618.0   \n",
       "160   Usable Directly Categorical      i         fiscal_year    68023.0   \n",
       "885   Usable Directly Categorical    NaN               GROUP   619129.0   \n",
       "276   Usable Directly Categorical      f                 Con     1518.0   \n",
       "860   Usable Directly Categorical      g     Police District    82958.0   \n",
       "36    Usable Directly Categorical      j      employer_phone   374362.0   \n",
       "\n",
       "      num of dist_val    % nans  max_val         mean  min_val      std_dev  \\\n",
       "1055             24.0  0.000000   2015.0  2003.500000   1992.0     6.922187   \n",
       "257              77.0  0.773605     78.0    41.061676      2.0    17.154842   \n",
       "403             182.0  0.000000      0.0     0.000000      0.0     0.000000   \n",
       "245            5997.0  0.958473      0.0     0.000000      0.0     0.000000   \n",
       "900             102.0  0.000573      0.0     0.000000      0.0     0.000000   \n",
       "160              67.0  0.002896   2016.0  2002.546811   1950.0    11.183397   \n",
       "885            2589.0  0.000000   9707.0   865.599382      0.0  1970.373963   \n",
       "276             158.0  0.000000      0.0     0.000000      0.0     0.000000   \n",
       "860              99.0  0.000241      0.0     0.000000      0.0     0.000000   \n",
       "36            49220.0  0.361428      0.0     0.000000      0.0     0.000000   \n",
       "\n",
       "                 sample_1              sample_2      sample_3      sample_4  \\\n",
       "1055                 1992                  1993          1994          1995   \n",
       "257                    39                    41            58            43   \n",
       "403   2010-05-02 00:00:00   2010-12-02 00:00:00      19/02/10      26/02/10   \n",
       "245            REQ0011118            REQ0011932    REQ0011476           nan   \n",
       "900                   590                   822           830           850   \n",
       "160                  2015                  2014          2013          2012   \n",
       "885                  1736                     0           928           557   \n",
       "276           Afghanistan  Afghanistan/Pakistan       Algeria        Angola   \n",
       "860                   690                   590           822           642   \n",
       "36                    nan            2625444811  662-254-7200  720-859-6111   \n",
       "\n",
       "                                               sample_5  \n",
       "1055                                               1996  \n",
       "257                                                  50  \n",
       "403                                 2010-05-03 00:00:00  \n",
       "245                                          REQ0013911  \n",
       "900                                                 871  \n",
       "160                                                2011  \n",
       "885                                                1961  \n",
       "276   Angola/Burundi/Central African Republic/Democr...  \n",
       "860                                                 620  \n",
       "36                                         206 391 7466  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abstained = pd.DataFrame(abstained, columns=cols)\n",
    "abstained.to_csv('results/cate_abstained.csv')\n",
    "abstained.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Usable Directly Numeric"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|Total|Matched|Mismatched| Abstained| Accuracy | Coverage|\n",
    "| --- | --- | --- | --- | --- | --- |\n",
    "|   5063  |  5055 | 3459  | 8| 63.84 | 99.84|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lfs_num = [lf_cast_to_numbers, lf_has_mean_std, lf_has_mean_dev_min,lf_max_min_difference,lf_sample_cast_to_float, lf_std_gt_mean]\n",
    "len(lfs_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labeling function names       \t       nan  category   numeric   extract  unusable   context\n",
      "lf_cast_to_numbers            \t         0      1092      5055        24       596      1747\n",
      "lf_has_mean_std               \t         0       965      5002        21       171      1663\n",
      "lf_has_mean_dev_min           \t         0       364      2496        21       142       673\n",
      "lf_max_min_difference         \t         0        14       958        13        86       400\n",
      "lf_sample_cast_to_float       \t         0         0      1865         2        20       346\n",
      "lf_std_gt_mean                \t         0       581      3435         8        23      1255\n"
     ]
    }
   ],
   "source": [
    "cms = test_lfs(lfs_num, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0, 1106, 5056, 24, 596, 1748],\n",
       " 7,\n",
       " 3474,\n",
       " 0.6372573733299722,\n",
       " 0.9986174205016789)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm, abstained, mismatched, accuracy, coverage = test_lfs_onecategory(lfs_num, df, 'Usable Directly Numeric')\n",
    "cm, len(abstained), len(mismatched), accuracy, coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 2087, 5063, 650, 891, 2050]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "catelens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1995</td>\n",
       "      <td>264.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>0.068182</td>\n",
       "      <td>2.274911e+01</td>\n",
       "      <td>1.929149e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.785024</td>\n",
       "      <td>2.138784453</td>\n",
       "      <td>0</td>\n",
       "      <td>0.016472597</td>\n",
       "      <td>4.424155104</td>\n",
       "      <td>0.197623255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>lvs5graze2</td>\n",
       "      <td>9597.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.380640</td>\n",
       "      <td>1.200000e+04</td>\n",
       "      <td>4.529727e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>155.666517</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1032</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>headcount</td>\n",
       "      <td>5807.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>n</td>\n",
       "      <td>vx</td>\n",
       "      <td>63992.0</td>\n",
       "      <td>63971.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.582761e+00</td>\n",
       "      <td>-6.970000e-05</td>\n",
       "      <td>-1.565658</td>\n",
       "      <td>0.407332</td>\n",
       "      <td>0.45098439</td>\n",
       "      <td>-0.95970893</td>\n",
       "      <td>-0.11458433</td>\n",
       "      <td>0.18220614</td>\n",
       "      <td>-0.064055555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>912</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>farmwork27</td>\n",
       "      <td>9597.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.997499</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.666667e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.471405</td>\n",
       "      <td>nan</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>nan</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>2799914.0</td>\n",
       "      <td>2799914.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.799913e+06</td>\n",
       "      <td>1.399956e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>808265.550800</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>h</td>\n",
       "      <td>outcome_hour</td>\n",
       "      <td>29421.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.300000e+01</td>\n",
       "      <td>1.400578e+01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.863203</td>\n",
       "      <td>16</td>\n",
       "      <td>18</td>\n",
       "      <td>17</td>\n",
       "      <td>14</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>i</td>\n",
       "      <td>q12JobCritCompensation</td>\n",
       "      <td>25090.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.336389e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.471512</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>othrhrcd1</td>\n",
       "      <td>9597.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.480671</td>\n",
       "      <td>3.520000e+02</td>\n",
       "      <td>1.031300e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.090769</td>\n",
       "      <td>0</td>\n",
       "      <td>nan</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>621</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>s1p1c2</td>\n",
       "      <td>9597.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.465145</td>\n",
       "      <td>5.600000e+01</td>\n",
       "      <td>2.477518e+01</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>15.341242</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>41</td>\n",
       "      <td>26</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            y_act Reason          Attribute_name  Total_val  \\\n",
       "44               Context_Specific    NaN                    1995      264.0   \n",
       "232              Context_Specific      a              lvs5graze2     9597.0   \n",
       "1032                     Unusable      l               headcount     5807.0   \n",
       "90               Context_Specific      n                      vx    63992.0   \n",
       "912              Context_Specific      a              farmwork27     9597.0   \n",
       "605                      Unusable      l              Unnamed: 0  2799914.0   \n",
       "32    Usable Directly Categorical      h            outcome_hour    29421.0   \n",
       "199   Usable Directly Categorical      i  q12JobCritCompensation    25090.0   \n",
       "338              Context_Specific      a               othrhrcd1     9597.0   \n",
       "621              Context_Specific      a                  s1p1c2     9597.0   \n",
       "\n",
       "      num of dist_val    % nans       max_val          mean   min_val  \\\n",
       "44              192.0  0.068182  2.274911e+01  1.929149e+00  0.000000   \n",
       "232              18.0  0.380640  1.200000e+04  4.529727e+00  0.000000   \n",
       "1032              1.0  0.000000  0.000000e+00  0.000000e+00  0.000000   \n",
       "90            63971.0  0.000000  1.582761e+00 -6.970000e-05 -1.565658   \n",
       "912               2.0  0.997499  2.000000e+00  1.666667e+00  1.000000   \n",
       "605         2799914.0  0.000000  2.799913e+06  1.399956e+06  0.000000   \n",
       "32               22.0  0.000000  2.300000e+01  1.400578e+01  0.000000   \n",
       "199               2.0  0.000000  1.000000e+00  3.336389e-01  0.000000   \n",
       "338              10.0  0.480671  3.520000e+02  1.031300e-01  0.000000   \n",
       "621              49.0  0.465145  5.600000e+01  2.477518e+01  2.000000   \n",
       "\n",
       "            std_dev     sample_1     sample_2     sample_3     sample_4  \\\n",
       "44         3.785024  2.138784453            0  0.016472597  4.424155104   \n",
       "232      155.666517            1            0            5            7   \n",
       "1032       0.000000            0            0            0            0   \n",
       "90         0.407332   0.45098439  -0.95970893  -0.11458433   0.18220614   \n",
       "912        0.471405          nan            2            1          nan   \n",
       "605   808265.550800            0            1            2            3   \n",
       "32         3.863203           16           18           17           14   \n",
       "199        0.471512            1            0            0            0   \n",
       "338        5.090769            0          nan           10            7   \n",
       "621       15.341242           14           12           41           26   \n",
       "\n",
       "          sample_5  \n",
       "44     0.197623255  \n",
       "232             12  \n",
       "1032             0  \n",
       "90    -0.064055555  \n",
       "912              2  \n",
       "605              4  \n",
       "32              19  \n",
       "199              0  \n",
       "338             35  \n",
       "621             33  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mismatched = pd.DataFrame(mismatched, columns=cols)\n",
    "mismatched.to_csv('results/num_mismatched.csv')\n",
    "mismatched.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>372</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>a</td>\n",
       "      <td>LOCATION</td>\n",
       "      <td>310579.0</td>\n",
       "      <td>285.0</td>\n",
       "      <td>0.059885</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>nan</td>\n",
       "      <td>(41.97562073966174, -87.72838313514907)</td>\n",
       "      <td>(41.90999558160536, -87.72639469321471)</td>\n",
       "      <td>(41.95302475075328, -87.78668323766634)</td>\n",
       "      <td>(41.83773267497158, -87.66589444999927)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>a</td>\n",
       "      <td>distance</td>\n",
       "      <td>258.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>28</td>\n",
       "      <td>12</td>\n",
       "      <td>11,2</td>\n",
       "      <td>12,9</td>\n",
       "      <td>18,5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>a</td>\n",
       "      <td>temp_inside</td>\n",
       "      <td>258.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.034884</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>21,5</td>\n",
       "      <td>22,5</td>\n",
       "      <td>20</td>\n",
       "      <td>nan</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>966</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>a</td>\n",
       "      <td>average_review_rating</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.001800</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.9 out of 5 stars</td>\n",
       "      <td>4.5 out of 5 stars</td>\n",
       "      <td>3.9 out of 5 stars</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>4.7 out of 5 stars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Main Source of Income</td>\n",
       "      <td>41544.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Wage/Salaries</td>\n",
       "      <td>Other sources of Income</td>\n",
       "      <td>Enterpreneurial Activities</td>\n",
       "      <td>Wage/Salaries</td>\n",
       "      <td>Wage/Salaries</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>NaN</td>\n",
       "      <td>install_size_kw_buckets</td>\n",
       "      <td>48722.0</td>\n",
       "      <td>48530.0</td>\n",
       "      <td>0.001190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>[[0171][5231][10114][1545][2016][252][352][451...</td>\n",
       "      <td>[[060][550][1024][1518][2023][2515][3012][3515...</td>\n",
       "      <td>[[0364][5313][1057][1513][207][258][305][351][...</td>\n",
       "      <td>[[0320][567][1020][1512][251][302][351][401][4...</td>\n",
       "      <td>[[0125][5219][10227][15124][2039][2520][3015][...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>NaN</td>\n",
       "      <td>location</td>\n",
       "      <td>188462.0</td>\n",
       "      <td>160187.0</td>\n",
       "      <td>0.013398</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>(37.7216893346731 -122.430482526944)</td>\n",
       "      <td>(37.8042860176436 -122.435150585742)</td>\n",
       "      <td>(37.7647299507493 -122.446671573844)</td>\n",
       "      <td>(37.7697365039652 -122.417760336994)</td>\n",
       "      <td>(37.7583327223052 -122.412354747671)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>892</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>a</td>\n",
       "      <td>educ37</td>\n",
       "      <td>9597.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.999687</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.942809</td>\n",
       "      <td>nan</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       y_act Reason           Attribute_name  Total_val  \\\n",
       "372  Usable Directly Numeric      a                 LOCATION   310579.0   \n",
       "574  Usable Directly Numeric      a                 distance      258.0   \n",
       "577  Usable Directly Numeric      a              temp_inside      258.0   \n",
       "966  Usable Directly Numeric      a    average_review_rating    10000.0   \n",
       "96   Usable Directly Numeric    NaN    Main Source of Income    41544.0   \n",
       "567  Usable Directly Numeric    NaN  install_size_kw_buckets    48722.0   \n",
       "774  Usable Directly Numeric    NaN                 location   188462.0   \n",
       "892  Usable Directly Numeric      a                   educ37     9597.0   \n",
       "\n",
       "     num of dist_val    % nans  max_val      mean  min_val   std_dev  \\\n",
       "372            285.0  0.059885      0.0  0.000000      0.0  0.000000   \n",
       "574            126.0  0.000000      0.0  0.000000      0.0  0.000000   \n",
       "577             11.0  0.034884      0.0  0.000000      0.0  0.000000   \n",
       "966             19.0  0.001800      0.0  0.000000      0.0  0.000000   \n",
       "96               3.0  0.000000      0.0  0.000000      0.0  0.000000   \n",
       "567          48530.0  0.001190      0.0  0.000000      0.0  0.000000   \n",
       "774         160187.0  0.013398      0.0  0.000000      0.0  0.000000   \n",
       "892              2.0  0.999687      2.0  0.666667      0.0  0.942809   \n",
       "\n",
       "                                              sample_1  \\\n",
       "372                                                nan   \n",
       "574                                                 28   \n",
       "577                                               21,5   \n",
       "966                                 4.9 out of 5 stars   \n",
       "96                                       Wage/Salaries   \n",
       "567  [[0171][5231][10114][1545][2016][252][352][451...   \n",
       "774               (37.7216893346731 -122.430482526944)   \n",
       "892                                                nan   \n",
       "\n",
       "                                              sample_2  \\\n",
       "372            (41.97562073966174, -87.72838313514907)   \n",
       "574                                                 12   \n",
       "577                                               22,5   \n",
       "966                                 4.5 out of 5 stars   \n",
       "96                             Other sources of Income   \n",
       "567  [[060][550][1024][1518][2023][2515][3012][3515...   \n",
       "774               (37.8042860176436 -122.435150585742)   \n",
       "892                                              False   \n",
       "\n",
       "                                              sample_3  \\\n",
       "372            (41.90999558160536, -87.72639469321471)   \n",
       "574                                               11,2   \n",
       "577                                                 20   \n",
       "966                                 3.9 out of 5 stars   \n",
       "96                          Enterpreneurial Activities   \n",
       "567  [[0364][5313][1057][1513][207][258][305][351][...   \n",
       "774               (37.7647299507493 -122.446671573844)   \n",
       "892                                                  2   \n",
       "\n",
       "                                              sample_4  \\\n",
       "372            (41.95302475075328, -87.78668323766634)   \n",
       "574                                               12,9   \n",
       "577                                                nan   \n",
       "966                                 5.0 out of 5 stars   \n",
       "96                                       Wage/Salaries   \n",
       "567  [[0320][567][1020][1512][251][302][351][401][4...   \n",
       "774               (37.7697365039652 -122.417760336994)   \n",
       "892                                              False   \n",
       "\n",
       "                                              sample_5  \n",
       "372            (41.83773267497158, -87.66589444999927)  \n",
       "574                                               18,5  \n",
       "577                                                 21  \n",
       "966                                 4.7 out of 5 stars  \n",
       "96                                       Wage/Salaries  \n",
       "567  [[0125][5219][10227][15124][2039][2520][3015][...  \n",
       "774               (37.7583327223052 -122.412354747671)  \n",
       "892                                                nan  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abstained = pd.DataFrame(abstained, columns=cols)\n",
    "abstained.to_csv('results/num_abstained.csv')\n",
    "abstained"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unusable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|Total|Matched|Mismatched| Abstained| accuracy | coverage|\n",
    "| --- | --- | --- | --- | --- | --- |\n",
    "|   891  |  856 | 414  | 70 | 66.48 | 92.14|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lfs_unusable = [lf_unusable, lf_unusable_nans]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labeling function names       \t       nan  category   numeric   extract  unusable   context\n",
      "lf_unusable                   \t         0         7       148        70       751       146\n",
      "lf_unusable_nans              \t         0        30         2         0       337        12\n"
     ]
    }
   ],
   "source": [
    "cms = test_lfs(lfs_unusable, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0, 37, 150, 70, 821, 157], 70, 414, 0.6647773279352227, 0.9214365881032548)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm, abstained, mismatched, accuracy, coverage = test_lfs_onecategory(lfs_unusable, df, 'Unusable')\n",
    "cm, len(abstained), len(mismatched), accuracy, coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 2087, 5063, 650, 891, 2050]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "catelens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>e</td>\n",
       "      <td>q26FrameLearnExpressJS</td>\n",
       "      <td>25090</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>Will Learn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Know</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>farmwork23</td>\n",
       "      <td>9597</td>\n",
       "      <td>2</td>\n",
       "      <td>99.301865</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.507463e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.999443e-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>i</td>\n",
       "      <td>q23FrameBackboneDotJS</td>\n",
       "      <td>25090</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yearly_sunlight_kwh_e</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.585561e+10</td>\n",
       "      <td>2.580733e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.228606e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>2420185385</td>\n",
       "      <td>4.49072e+07</td>\n",
       "      <td>6054636897</td>\n",
       "      <td>1020895160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Usable Directly Categorical</td>\n",
       "      <td>e</td>\n",
       "      <td>ji_live_in_dom_svc_contract</td>\n",
       "      <td>374362</td>\n",
       "      <td>3</td>\n",
       "      <td>99.359444</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SUPPORT_SERVICES_EXPENDITURE</td>\n",
       "      <td>1224</td>\n",
       "      <td>1224</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.350185e+07</td>\n",
       "      <td>2.631397e+06</td>\n",
       "      <td>139963.000000</td>\n",
       "      <td>3.280875e+06</td>\n",
       "      <td>735036</td>\n",
       "      <td>350902</td>\n",
       "      <td>1007732</td>\n",
       "      <td>483488</td>\n",
       "      <td>8520926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>NaN</td>\n",
       "      <td>lng_avg</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-6.988566e+01</td>\n",
       "      <td>-9.364726e+01</td>\n",
       "      <td>-157.507901</td>\n",
       "      <td>1.899155e+01</td>\n",
       "      <td>-116.635</td>\n",
       "      <td>-86.8507</td>\n",
       "      <td>-148.508</td>\n",
       "      <td>-111.895</td>\n",
       "      <td>-92.6524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>s1p2c5</td>\n",
       "      <td>9597</td>\n",
       "      <td>24</td>\n",
       "      <td>99.322705</td>\n",
       "      <td>5.600000e+01</td>\n",
       "      <td>3.413846e+01</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.729907e+01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33</td>\n",
       "      <td>29</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>n</td>\n",
       "      <td>z</td>\n",
       "      <td>63984</td>\n",
       "      <td>63959</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.184194e+01</td>\n",
       "      <td>1.574555e-03</td>\n",
       "      <td>-19.199339</td>\n",
       "      <td>8.976619e-01</td>\n",
       "      <td>0.0671728</td>\n",
       "      <td>0.108988</td>\n",
       "      <td>-0.0105302</td>\n",
       "      <td>-0.0895059</td>\n",
       "      <td>-0.284885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>Usable Directly Numeric</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yearly_sunlight_kwh_n</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.215200e+10</td>\n",
       "      <td>1.055046e+09</td>\n",
       "      <td>4745.559338</td>\n",
       "      <td>2.273631e+09</td>\n",
       "      <td>4745.56</td>\n",
       "      <td>1128360422</td>\n",
       "      <td>2.18659e+06</td>\n",
       "      <td>1780582587</td>\n",
       "      <td>455410026</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           y_act Reason                Attribute_name  \\\n",
       "251  Usable Directly Categorical      e        q26FrameLearnExpressJS   \n",
       "515             Context_Specific      a                    farmwork23   \n",
       "247  Usable Directly Categorical      i         q23FrameBackboneDotJS   \n",
       "208      Usable Directly Numeric    NaN         yearly_sunlight_kwh_e   \n",
       "0    Usable Directly Categorical      e   ji_live_in_dom_svc_contract   \n",
       "238      Usable Directly Numeric    NaN  SUPPORT_SERVICES_EXPENDITURE   \n",
       "194      Usable Directly Numeric    NaN                       lng_avg   \n",
       "24              Context_Specific      a                        s1p2c5   \n",
       "325             Context_Specific      n                             z   \n",
       "206      Usable Directly Numeric    NaN         yearly_sunlight_kwh_n   \n",
       "\n",
       "     Total_val  num of dist_val     % nans       max_val          mean  \\\n",
       "251      25090                2        NaN  0.000000e+00  0.000000e+00   \n",
       "515       9597                2  99.301865  2.000000e+00  1.507463e+00   \n",
       "247      25090                3        NaN  0.000000e+00  0.000000e+00   \n",
       "208         52               52        NaN  2.585561e+10  2.580733e+09   \n",
       "0       374362                3  99.359444  0.000000e+00  0.000000e+00   \n",
       "238       1224             1224        NaN  2.350185e+07  2.631397e+06   \n",
       "194         52               52        NaN -6.988566e+01 -9.364726e+01   \n",
       "24        9597               24  99.322705  5.600000e+01  3.413846e+01   \n",
       "325      63984            63959        NaN  2.184194e+01  1.574555e-03   \n",
       "206         52               52        NaN  1.215200e+10  1.055046e+09   \n",
       "\n",
       "           min_val       std_dev    sample_1    sample_2     sample_3  \\\n",
       "251       0.000000  0.000000e+00  Will Learn         NaN         Know   \n",
       "515       1.000000  4.999443e-01         NaN           2            1   \n",
       "247       0.000000  0.000000e+00         NaN           0            1   \n",
       "208       0.000000  4.228606e+09           0  2420185385  4.49072e+07   \n",
       "0         0.000000  0.000000e+00         NaN           A            Y   \n",
       "238  139963.000000  3.280875e+06      735036      350902      1007732   \n",
       "194    -157.507901  1.899155e+01    -116.635    -86.8507     -148.508   \n",
       "24        4.000000  1.729907e+01         NaN          33           29   \n",
       "325     -19.199339  8.976619e-01   0.0671728    0.108988   -0.0105302   \n",
       "206    4745.559338  2.273631e+09     4745.56  1128360422  2.18659e+06   \n",
       "\n",
       "       sample_4    sample_5  \n",
       "251         NaN         NaN  \n",
       "515         NaN           2  \n",
       "247         NaN         NaN  \n",
       "208  6054636897  1020895160  \n",
       "0             N           N  \n",
       "238      483488     8520926  \n",
       "194    -111.895    -92.6524  \n",
       "24           14          12  \n",
       "325  -0.0895059   -0.284885  \n",
       "206  1780582587   455410026  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mismatched = pd.DataFrame(mismatched, columns=cols)\n",
    "mismatched.to_csv('results/unusable_mismatched.csv')\n",
    "mismatched.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>q23FramePadrino</td>\n",
       "      <td>25090</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Padrino</td>\n",
       "      <td>Padrino</td>\n",
       "      <td>Padrino</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>ZoneDist4</td>\n",
       "      <td>324403</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>R5</td>\n",
       "      <td>M3-1</td>\n",
       "      <td>R7B</td>\n",
       "      <td>R2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>k</td>\n",
       "      <td>ad7521</td>\n",
       "      <td>9597</td>\n",
       "      <td>7</td>\n",
       "      <td>23.997082</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>intercropping</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>k</td>\n",
       "      <td>isbn</td>\n",
       "      <td>10000</td>\n",
       "      <td>9300</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>439023483</td>\n",
       "      <td>439554934</td>\n",
       "      <td>316015849</td>\n",
       "      <td>61120081</td>\n",
       "      <td>743273567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>q22LangProfPython</td>\n",
       "      <td>25090</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Python</td>\n",
       "      <td>Python</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>q23FrameStruts</td>\n",
       "      <td>25090</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Struts</td>\n",
       "      <td>Struts</td>\n",
       "      <td>Struts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>ZoneDist4</td>\n",
       "      <td>89830</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>R5</td>\n",
       "      <td>R6A</td>\n",
       "      <td>R5</td>\n",
       "      <td>R6A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>q21CoreCompDatabaseDesign</td>\n",
       "      <td>25090</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Database Design</td>\n",
       "      <td>Database Design</td>\n",
       "      <td>Database Design</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>q17HirChaNoDiversCandidates</td>\n",
       "      <td>25090</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Not enough diverse candidates</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Not enough diverse candidates</td>\n",
       "      <td>Not enough diverse candidates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>k</td>\n",
       "      <td>Sentence_Index</td>\n",
       "      <td>662207</td>\n",
       "      <td>27</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>3.697087</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.129019</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       y_act Reason               Attribute_name  Total_val  num of dist_val  \\\n",
       "32  Unusable      l              q23FramePadrino      25090                2   \n",
       "16  Unusable      l                    ZoneDist4     324403                6   \n",
       "7   Unusable      k                       ad7521       9597                7   \n",
       "10  Unusable      k                         isbn      10000             9300   \n",
       "27  Unusable      l            q22LangProfPython      25090                2   \n",
       "30  Unusable      l               q23FrameStruts      25090                2   \n",
       "14  Unusable      l                    ZoneDist4      89830                2   \n",
       "25  Unusable      l    q21CoreCompDatabaseDesign      25090                2   \n",
       "24  Unusable      l  q17HirChaNoDiversCandidates      25090                2   \n",
       "3   Unusable      k               Sentence_Index     662207               27   \n",
       "\n",
       "       % nans  max_val      mean  min_val   std_dev   sample_1  \\\n",
       "32        NaN      0.0  0.000000      0.0  0.000000        NaN   \n",
       "16        NaN      0.0  0.000000      0.0  0.000000        NaN   \n",
       "7   23.997082      0.0  0.000000      0.0  0.000000          0   \n",
       "10   7.000000      0.0  0.000000      0.0  0.000000  439023483   \n",
       "27        NaN      0.0  0.000000      0.0  0.000000        NaN   \n",
       "30        NaN      0.0  0.000000      0.0  0.000000        NaN   \n",
       "14        NaN      0.0  0.000000      0.0  0.000000        NaN   \n",
       "25        NaN      0.0  0.000000      0.0  0.000000        NaN   \n",
       "24        NaN      0.0  0.000000      0.0  0.000000        NaN   \n",
       "3    0.000000     26.0  3.697087      0.0  3.129019          0   \n",
       "\n",
       "                         sample_2         sample_3  \\\n",
       "32                            NaN          Padrino   \n",
       "16                             R5             M3-1   \n",
       "7                               1                0   \n",
       "10                      439554934        316015849   \n",
       "27                            NaN           Python   \n",
       "30                            NaN           Struts   \n",
       "14                             R5              R6A   \n",
       "25                            NaN  Database Design   \n",
       "24  Not enough diverse candidates              NaN   \n",
       "3                               1                2   \n",
       "\n",
       "                         sample_4                       sample_5  \n",
       "32                        Padrino                        Padrino  \n",
       "16                            R7B                             R2  \n",
       "7                               1                  intercropping  \n",
       "10                       61120081                      743273567  \n",
       "27                         Python                         Python  \n",
       "30                         Struts                         Struts  \n",
       "14                             R5                            R6A  \n",
       "25                Database Design                Database Design  \n",
       "24  Not enough diverse candidates  Not enough diverse candidates  \n",
       "3                               3                              4  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abstained = pd.DataFrame(abstained, columns=cols)\n",
    "abstained.to_csv('results/unusable_abstained.csv')\n",
    "abstained.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Context_Specific"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lf_unusable(row):\n",
    "    if lf_unusable_dist(row) or lf_unusable_nans(row):\n",
    "        return 4\n",
    "    return 0  \n",
    "\n",
    "def lf_unusable_dist(row):\n",
    "    # Case k. A number indicating the position of a record in its dataset table, e.g., serial number\n",
    "    # Case l. An attribute that is likely the primary key in its dataset table, or an attribute whose values will almost surely be unique for all records in its dataset table but is not a numeric feature\n",
    "    # PRIORITY: 1\n",
    "    # 1. # distinct == 1\n",
    "    # 2. % distinct == 100%\n",
    "    # 3. % nans     > 99%\n",
    "    # [TODO] total_nans seems to be a loose contrain here\n",
    "    # [51, 191, 80, 786, 219, 0]\n",
    "    total_val = row['Total_val']\n",
    "    dist_val = row['num of dist_val']\n",
    "    total_nans = row['% nans']\n",
    "    if dist_val <= 1 or (dist_val/total_val) > 0.99 or total_nans > 99:\n",
    "        return 4\n",
    "    return 0\n",
    "\n",
    "\n",
    "def lf_unusable_nans(row):\n",
    "    # Case k. A number indicating the position of a record in its dataset table, e.g., serial number\n",
    "    # Case l. An attribute that is likely the primary key in its dataset table, or an attribute whose values will almost surely be unique for all records in its dataset table but is not a numeric feature\n",
    "    # PRIORITY: 1\n",
    "    # 1. samples are of NaN values\n",
    "    # [TODO] % nans should be more precise here, while some of them are not available\n",
    "    # [30, 2, 0, 337, 12, 0]\n",
    "    def check_nan(string):\n",
    "        return string == 'nan'\n",
    "\n",
    "    samples = get_samples(row)\n",
    "    nans_in_samples = [check_nan(sample) for sample in samples]\n",
    "    if sum(nans_in_samples) >= 3:\n",
    "        return 4\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def lf_context_specific(row):\n",
    "    if lf_unusable(row):\n",
    "        return 0\n",
    "    if lf_key_word_id(row, False):\n",
    "        return 5\n",
    "    return 0\n",
    "    \n",
    "def lf_key_word_id(row, single=True):\n",
    "    # a key word in attribute name while has different representation from classes\n",
    "    # PRIORITY: 2\n",
    "    # e.g. student_id  studentID\n",
    "    # 1. attribute name contains id\n",
    "    \n",
    "    if single and lf_unusable(row):\n",
    "        return 0\n",
    "    \n",
    "    string = str(row['Attribute_name'])\n",
    "    parts = re.split('[^a-zA-Z0-9]', string)\n",
    "    for part in parts:\n",
    "        if part.lower() == 'id':\n",
    "            return 5\n",
    "    if re.findall('ID', string):\n",
    "        return 5\n",
    "    return 0\n",
    "\n",
    "def lf_undestandable_name(row, single=True):\n",
    "    # if the attribute name is not human understandable, mark it as context specfic\n",
    "    # PRIORITY: 2\n",
    "    # e.g. CB2010, x, xy\n",
    "    # 1. split the attribute name into individual parts\n",
    "    \n",
    "    if single and lf_unusable(row):\n",
    "        return 0\n",
    "    \n",
    "    string = str(row['Attribute_name']).lower()\n",
    "    first_name = re.search('[A-Z]?[a-z]*', 'n.programs').group()\n",
    "    if first_name == '1':\n",
    "        return 5\n",
    "    if first_name.lower() not in words.words():\n",
    "        return 5\n",
    "    return 0\n",
    "\n",
    "def lf_ambiguous_terms(row, single=True):\n",
    "    # if the attribute name is a single ambiguous term\n",
    "    # PRIORITY: 2\n",
    "    # e.g. name, id\n",
    "    \n",
    "    if single and lf_unusable(row):\n",
    "        return 0\n",
    "    \n",
    "    if str(row['Attribute_name']).lower() == 'id' or str(row['Attribute_name']).lower() == 'name':\n",
    "        return 5\n",
    "    if str(row['Attribute_name']).lower()[-2:] == 'id' or str(row['Attribute_name']).lower()[-4:] == 'name':\n",
    "        return 5\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'n'"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.search('[A-Z]?[a-z]*', 'n.programs').group()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "lfs_cs = [lf_undestandable_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-210-6a16b85f0a71>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_lfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlfs_cs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/Documents/Project/CSE291/lfs/helper.py\u001b[0m in \u001b[0;36mtest_lfs\u001b[0;34m(lfs, df, verbose)\u001b[0m\n\u001b[1;32m     76\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlfs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m             \u001b[0mlf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mret\u001b[0m\u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m                 \u001b[0mcms\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my_act\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-206-f3249960192f>\u001b[0m in \u001b[0;36mlf_undestandable_name\u001b[0;34m(row, single)\u001b[0m\n\u001b[1;32m     39\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfirst_name\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'1'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0mfirst_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mwords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     42\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.5/site-packages/nltk/corpus/reader/wordlist.py\u001b[0m in \u001b[0;36mwords\u001b[0;34m(self, fileids, ignore_lines_startswith)\u001b[0m\n\u001b[1;32m     20\u001b[0m     \"\"\"\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfileids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_lines_startswith\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m         return [line for line in line_tokenize(self.raw(fileids))\n\u001b[0m\u001b[1;32m     23\u001b[0m                 if not line.startswith(ignore_lines_startswith)]\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.5/site-packages/nltk/corpus/reader/wordlist.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfileids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_lines_startswith\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         return [line for line in line_tokenize(self.raw(fileids))\n\u001b[0;32m---> 23\u001b[0;31m                 if not line.startswith(ignore_lines_startswith)]\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfileids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "cms = test_lfs(lfs_cs, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>lvs3graze1</td>\n",
       "      <td>9597</td>\n",
       "      <td>12</td>\n",
       "      <td>42.752944</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.396250</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.016422</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Team</td>\n",
       "      <td>2150</td>\n",
       "      <td>297</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1463.0</td>\n",
       "      <td>1291.615814</td>\n",
       "      <td>1102.0</td>\n",
       "      <td>103.268764</td>\n",
       "      <td>1207</td>\n",
       "      <td>1210</td>\n",
       "      <td>1228</td>\n",
       "      <td>1260</td>\n",
       "      <td>1374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>794</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>s2p1c3qharv</td>\n",
       "      <td>9597</td>\n",
       "      <td>204</td>\n",
       "      <td>63.665729</td>\n",
       "      <td>8000000.0</td>\n",
       "      <td>7353.813651</td>\n",
       "      <td>0.0</td>\n",
       "      <td>172758.422900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>90</td>\n",
       "      <td>105</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>916</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>s2p2c6sold</td>\n",
       "      <td>9597</td>\n",
       "      <td>5</td>\n",
       "      <td>73.887673</td>\n",
       "      <td>300.0</td>\n",
       "      <td>0.323224</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.376567</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>200</td>\n",
       "      <td>300</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>631</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>o</td>\n",
       "      <td>CODIGO_COR_RACA</td>\n",
       "      <td>21124</td>\n",
       "      <td>5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.813482</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.956523</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1042</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENROLL</td>\n",
       "      <td>359836</td>\n",
       "      <td>23399</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1077381.0</td>\n",
       "      <td>3123.834339</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14008.465000</td>\n",
       "      <td>7568</td>\n",
       "      <td>19961</td>\n",
       "      <td>2045</td>\n",
       "      <td>2972</td>\n",
       "      <td>3539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1063</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>hm15owner</td>\n",
       "      <td>9597</td>\n",
       "      <td>4</td>\n",
       "      <td>68.219235</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.236393</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.486729</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ADM1DHS</td>\n",
       "      <td>4861</td>\n",
       "      <td>28</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9999.0</td>\n",
       "      <td>1945.311664</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3948.520276</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>70</td>\n",
       "      <td>23</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>s1p2c4area</td>\n",
       "      <td>9597</td>\n",
       "      <td>41</td>\n",
       "      <td>98.582890</td>\n",
       "      <td>70.0</td>\n",
       "      <td>10.446544</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.809756</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1061</th>\n",
       "      <td>Context_Specific</td>\n",
       "      <td>a</td>\n",
       "      <td>hm15</td>\n",
       "      <td>9597</td>\n",
       "      <td>14</td>\n",
       "      <td>58.518287</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>0.576488</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.074431</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 y_act Reason   Attribute_name  Total_val  num of dist_val  \\\n",
       "211   Context_Specific      a       lvs3graze1       9597               12   \n",
       "165   Context_Specific    NaN             Team       2150              297   \n",
       "794   Context_Specific      a      s2p1c3qharv       9597              204   \n",
       "916   Context_Specific      a       s2p2c6sold       9597                5   \n",
       "631   Context_Specific      o  CODIGO_COR_RACA      21124                5   \n",
       "1042  Context_Specific    NaN           ENROLL     359836            23399   \n",
       "1063  Context_Specific      a        hm15owner       9597                4   \n",
       "13    Context_Specific    NaN          ADM1DHS       4861               28   \n",
       "728   Context_Specific      a       s1p2c4area       9597               41   \n",
       "1061  Context_Specific      a             hm15       9597               14   \n",
       "\n",
       "         % nans    max_val         mean  min_val        std_dev sample_1  \\\n",
       "211   42.752944       12.0     0.396250      0.0       2.016422        0   \n",
       "165         NaN     1463.0  1291.615814   1102.0     103.268764     1207   \n",
       "794   63.665729  8000000.0  7353.813651      0.0  172758.422900      NaN   \n",
       "916   73.887673      300.0     0.323224      0.0       8.376567      NaN   \n",
       "631    0.000000        5.0     1.813482      1.0       0.956523        3   \n",
       "1042        NaN  1077381.0  3123.834339      0.0   14008.465000     7568   \n",
       "1063  68.219235        3.0     0.236393      0.0       0.486729      NaN   \n",
       "13          NaN     9999.0  1945.311664      1.0    3948.520276        8   \n",
       "728   98.582890       70.0    10.446544      0.0       9.809756        5   \n",
       "1061  58.518287     1000.0     0.576488      0.0      16.074431        0   \n",
       "\n",
       "     sample_2 sample_3 sample_4 sample_5  \n",
       "211       NaN       12        3        9  \n",
       "165      1210     1228     1260     1374  \n",
       "794       100       90      105      200  \n",
       "916         0      200      300       50  \n",
       "631         1        2        4        5  \n",
       "1042    19961     2045     2972     3539  \n",
       "1063        3        1        2        0  \n",
       "13         13       70       23       68  \n",
       "728        15      NaN       14       12  \n",
       "1061      NaN       10        1        3  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[(df.y_act=='Context_Specific')].sample(10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3.5",
   "language": "python",
   "name": "python3.5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}