check-errors-git.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "\n",
    "import kuzushiji.viz\n",
    "from kuzushiji.data_utils import get_encoded_classes, load_train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = load_train_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('_runs/clf-resnet152/errors.csv.gz')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(df['pred'] == df['true']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(df['true'] == 'seg_fp').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(df['pred'] == 'seg_fp').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(df[df['true'] == 'seg_fp']['pred'] == 'seg_fp').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(df[df['pred'] == 'seg_fp']['true'] == 'seg_fp').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_true_no_seg_fp = df[df['true'] != 'seg_fp']\n",
    "(df_true_no_seg_fp['true'] == df_true_no_seg_fp['pred']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "err_items = df[df['pred'] != df['true']]\n",
    "err_items_chars = err_items[(err_items['true'] != 'seg_fp') & (err_items['pred'] != 'seg_fp')]\n",
    "err_items_seg_fp_fn = err_items[err_items['true'] == 'seg_fp']\n",
    "err_items_seg_fp_fp = err_items[err_items['pred'] == 'seg_fp']\n",
    "print('Error kinds:')\n",
    "print(f'{len(err_items_chars) / len(df):.2%} true char != pred char')\n",
    "print(f'{len(err_items_seg_fp_fn + err_items_seg_fp_fp) / len(df):.2%} with segfp')\n",
    "print(f'{len(err_items_seg_fp_fn) / len(df):.2%} true segfp != pred char')\n",
    "print(f'{len(err_items_seg_fp_fp) / len(df):.2%} true char != pred segfp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_error_count = df[df['pred'] != df['true']].groupby('image_id')['pred'].count().sort_values(ascending=False)\n",
    "image_error_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freqs = Counter()\n",
    "for label in df_train['labels']:\n",
    "    for i, c in enumerate(label.split()):\n",
    "        if i % 5 == 0:\n",
    "            freqs[c] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls_by_id = {id: cls for cls, id in get_encoded_classes().items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def analyze_errors(df, only_errors=False, n=50):\n",
    "    if only_errors:\n",
    "        df = df[df['true'] != df['pred']]\n",
    "    df = df.sample(n=n, random_state=42)\n",
    "    data = []\n",
    "    for item in df.itertuples():\n",
    "        top_logits_list = list(map(float, item.top_k_logits.split()))\n",
    "        top_logits = dict(zip([cls_by_id[int(id)] for id in item.top_k_classes.split()],\n",
    "                              top_logits_list))\n",
    "        places = {cls: idx + 1 for idx, (cls, _) in enumerate(sorted(top_logits.items(), key=lambda x: -x[1]))}\n",
    "        data.append({\n",
    "            'true': item.true,\n",
    "            'pred': item.pred,\n",
    "            'true_score': top_logits.get(item.true, 0),\n",
    "            'pred_score': top_logits[item.pred],\n",
    "            'second_score': top_logits_list[1],\n",
    "            'true_place': places.get(item.true, len(places)),\n",
    "            'true_freq': freqs.get(item.true, 0),\n",
    "            'pred_freq': freqs.get(item.pred, 0),\n",
    "        })\n",
    "    # TODO:\n",
    "    # - true/pred example crops\n",
    "    return pd.DataFrame(data)\n",
    "\n",
    "#analyze_errors(df[df['image_id'] == '200003076_00149_1'], only_errors=True)\n",
    "analyze_errors(df, only_errors=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze_errors(df[df['pred'] == 'seg_fp'], only_errors=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze_errors(df[df['pred'] == 'seg_fp'], only_errors=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viz_errors(image_id, with_true_boxes=True):\n",
    "    image, title = kuzushiji.viz.visualize_clf_errors(image_id, df)\n",
    "    if with_true_boxes:\n",
    "        true_boxes = (\n",
    "            np.array(df_train[df_train['image_id'] == image_id].iloc[0].labels.split())\n",
    "            .reshape(-1, 5)[:, 1:].astype(int))\n",
    "        image = kuzushiji.viz.visualize_boxes(image, true_boxes, thickness=2, color=(0, 0, 0))\n",
    "    print(title)\n",
    "    plt.figure(figsize=(20, 20))\n",
    "    plt.title(title)\n",
    "    plt.imshow(image)\n",
    "    \n",
    "viz_errors('200003076_00149_1', with_true_boxes=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for image_id in image_error_count[:10].index:\n",
    "    viz_errors(image_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "rng = np.random.RandomState(42)\n",
    "for image_id in rng.choice(sorted(set(df['image_id'].values)), 10, replace=False):\n",
    "    viz_errors(image_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}