agglomerative_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "e081740e-8169-4481-b1df-f5dd5488314f",
    "_uuid": "0bee86255243664f24e4bcf48af2228a3100a8b7"
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import timeit\n",
    "\n",
    "from trackml.dataset import load_event, load_dataset\n",
    "from trackml.score import score_event\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import hdbscan\n",
    "from scipy import stats\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.cluster import DBSCAN, AgglomerativeClustering, AffinityPropagation\n",
    "\n",
    "def create_one_event_submission(event_id, hits, labels):\n",
    "    sub_data = np.column_stack(([event_id]*len(hits), hits.hit_id.values, labels))\n",
    "    submission = pd.DataFrame(data=sub_data, columns=[\"event_id\", \"hit_id\", \"track_id\"]).astype(int)\n",
    "    return submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_cell_guid": "572fcbb6-8c7b-4a09-8916-8ec76689130f",
    "_uuid": "63414de98667e95f60407c9155899a25a321cffc"
   },
   "outputs": [],
   "source": [
    "# Change this according to your directory preferred setting\n",
    "path_to_train = \"./data/train_1\"\n",
    "\n",
    "# This event is in Train_1\n",
    "event_prefix = \"event000001000\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_cell_guid": "e06d1ed7-5091-4d67-abb4-5984b137e2e6",
    "_uuid": "c2f70ae63abffcc09a534bb17fb89df8ffddb722",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class Clusterer(object):\n",
    "    def __init__(self,rz_scales=[0.65, 0.965, 1.528], eps=0.0035):                        \n",
    "        self.rz_scales=rz_scales\n",
    "        self.epsilon = eps\n",
    "        \n",
    "    def _eliminate_outliers(self,labels,M):\n",
    "        norms=np.zeros((len(labels)),np.float32)\n",
    "        indices=np.zeros((len(labels)),np.float32)\n",
    "        for i, cluster in tqdm(enumerate(labels),total=len(labels)):\n",
    "            if cluster == 0:\n",
    "                continue\n",
    "            index = np.argwhere(self.clusters==cluster)\n",
    "            index = np.reshape(index,(index.shape[0]))\n",
    "            indices[i] = len(index)\n",
    "            x = M[index]\n",
    "            norms[i] = self._test_quadric(x)\n",
    "        threshold1 = np.percentile(norms,90)*5\n",
    "        threshold2 = 25\n",
    "        threshold3 = 6\n",
    "        for i, cluster in enumerate(labels):\n",
    "            if norms[i] > threshold1 or indices[i] > threshold2 or indices[i] < threshold3:\n",
    "                self.clusters[self.clusters==cluster]=0   \n",
    "    \n",
    "    def _test_quadric(self,x):\n",
    "        if x.size == 0 or len(x.shape)<2:\n",
    "            return 0\n",
    "        Z = np.zeros((x.shape[0],10), np.float32)\n",
    "        Z[:,0] = x[:,0]**2\n",
    "        Z[:,1] = 2*x[:,0]*x[:,1]\n",
    "        Z[:,2] = 2*x[:,0]*x[:,2]\n",
    "        Z[:,3] = 2*x[:,0]\n",
    "        Z[:,4] = x[:,1]**2\n",
    "        Z[:,5] = 2*x[:,1]*x[:,2]\n",
    "        Z[:,6] = 2*x[:,1]\n",
    "        Z[:,7] = x[:,2]**2\n",
    "        Z[:,8] = 2*x[:,2]\n",
    "        Z[:,9] = 1\n",
    "        v, s, t = np.linalg.svd(Z,full_matrices=False)        \n",
    "        smallest_index = np.argmin(np.array(s))\n",
    "        T = np.array(t)\n",
    "        T = T[smallest_index,:]        \n",
    "        norm = np.linalg.norm(np.dot(Z,T), ord=2)**2\n",
    "        return norm\n",
    "\n",
    "    def _preprocess(self, hits):\n",
    "        \n",
    "        x = hits.x.values\n",
    "        y = hits.y.values\n",
    "        z = hits.z.values\n",
    "\n",
    "        r = np.sqrt(x**2 + y**2 + z**2)\n",
    "        hits['x2'] = x/r\n",
    "        hits['y2'] = y/r\n",
    "\n",
    "        r = np.sqrt(x**2 + y**2)\n",
    "        hits['z2'] = z/r\n",
    "\n",
    "        ss = StandardScaler()\n",
    "        X = ss.fit_transform(hits[['x2', 'y2', 'z2']].values)\n",
    "        for i, rz_scale in enumerate(self.rz_scales):\n",
    "            X[:,i] = X[:,i] * rz_scale\n",
    "       \n",
    "        return X\n",
    "    \n",
    "    def _init(self,dfh):\n",
    "        dfh['r'] = np.sqrt(dfh['x'].values**2+dfh['y'].values**2+dfh['z'].values**2)\n",
    "        dfh['rt'] = np.sqrt(dfh['x'].values**2+dfh['y'].values**2)\n",
    "        dfh['a0'] = np.arctan2(dfh['y'].values,dfh['x'].values)\n",
    "        dfh['z1'] = dfh['z'].values/dfh['rt'].values\n",
    "        dfh['x2'] = 1/dfh['z1'].values\n",
    "        dz0 = -0.00070\n",
    "        stepdz = 0.00001\n",
    "        stepeps = 0.000005\n",
    "        mm = 1\n",
    "        for ii in tqdm(range(100)):\n",
    "            mm = mm*(-1)\n",
    "            dz = mm*(dz0+ii*stepdz)\n",
    "            dfh['a1'] = dfh['a0'].values+dz*dfh['z'].values*np.sign(dfh['z'].values)\n",
    "            dfh['sina1'] = np.sin(dfh['a1'].values)\n",
    "            dfh['cosa1'] = np.cos(dfh['a1'].values)\n",
    "            dfh['x1'] = dfh['a1'].values/dfh['z1'].values\n",
    "            ss = StandardScaler()\n",
    "            dfs = ss.fit_transform(dfh[['sina1','cosa1','z1','x1','x2']].values)\n",
    "            cx = np.array([1, 1, 0.75, 0.5, 0.5])\n",
    "            dfs = np.multiply(dfs, cx)\n",
    "\n",
    "#             clusters=DBSCAN(eps=self.epsilon+ii*stepeps,min_samples=1,metric='euclidean', n_jobs=1).fit(dfs).labels_            \n",
    "            clusters = AgglomerativeClustering().fit(dfs).labels_\n",
    "            if ii==0:\n",
    "                dfh['s1'] = clusters\n",
    "                dfh['N1'] = dfh.groupby('s1')['s1'].transform('count')\n",
    "            else:\n",
    "                dfh['s2'] = clusters\n",
    "                dfh['N2'] = dfh.groupby('s2')['s2'].transform('count')\n",
    "                maxs1 = dfh['s1'].max()\n",
    "                cond = np.where((dfh['N2'].values>dfh['N1'].values) & (dfh['N2'].values<20))\n",
    "                s1 = dfh['s1'].values\n",
    "                s1[cond] = dfh['s2'].values[cond]+maxs1\n",
    "                dfh['s1'] = s1\n",
    "                dfh['s1'] = dfh['s1'].astype('int64')\n",
    "                dfh['N1'] = dfh.groupby('s1')['s1'].transform('count')\n",
    "                \n",
    "        return dfh['s1'].values    \n",
    "    \n",
    "    def predict(self, hits):    \n",
    "        self.clusters = self._init(hits) \n",
    "        X = self._preprocess(hits) \n",
    "        cl = hdbscan.HDBSCAN(min_samples=1,min_cluster_size=7,\n",
    "                             metric='braycurtis',cluster_selection_method='leaf',algorithm='best', leaf_size=50)\n",
    "        \n",
    "#         cl = AgglomerativeClustering(n_clusters=len(np.unique(self.clusters)), linkage=\"ward\")\n",
    "        labels = np.unique(self.clusters)\n",
    "        self._eliminate_outliers(labels,X)\n",
    "        n_labels = 0\n",
    "        while n_labels < len(labels):\n",
    "            n_labels = len(labels)            \n",
    "            max_len = np.max(self.clusters)\n",
    "            mask = self.clusters == 0\n",
    "            self.clusters[mask] = cl.fit_predict(X[mask])+max_len\n",
    "        return self.clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_cell_guid": "2b20cf0c-2754-48dd-ab1e-4f489c2aa05c",
    "_uuid": "7f8de52b9022581bf10aa813d2db005b842f0be7",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path_to_test = \"./data/test\"\n",
    "eps = 0.0035\n",
    "\n",
    "def one_loop(event_id):\n",
    "    hits  = pd.read_csv(path_to_test + '/event%s-hits.csv'%event_id)\n",
    "    cells = pd.read_csv(path_to_test + '/event%s-cells.csv'%event_id)\n",
    "    print('Event ID: ', event_id)\n",
    "                \n",
    "    # Track pattern recognition \n",
    "    model = Clusterer(eps=eps)\n",
    "    labels = model.predict(hits)\n",
    "\n",
    "    # Prepare submission for an event\n",
    "    one_submission = create_one_event_submission(event_id, hits, labels)\n",
    "    one_submission.to_csv('./%09d.helix.csv.gz'%int(event_id), index=False, compression='gzip')\n",
    "            \n",
    "    return one_submission\n",
    "\n",
    "def create_test_submissions(path_to_test = \"./data/test\", start=0, end=125):\n",
    "    event_ids = [ '%09d'%i for i in range(start,end) ]\n",
    "\n",
    "    pool = Pool(processes=14)\n",
    "    results = pool.map(one_loop, event_ids)\n",
    "    pool.close()\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  000000033\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6dfa488ae79490a9dc338cde6118d52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  000000003\n",
      "Event ID:  000000009\n",
      "Event ID:  000000036\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e85fc14570ca4949be044f18cb049960",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  000000015\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0cb40d20bd384c2b87123623bcdc2a9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04808d481daa4bb4a9bad4bbf8de6039",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  000000018\n",
      "Event ID:  000000027\n",
      "Event ID:  000000012\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4967fc7cc9d4da8b34ce4505576fffe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  000000021\n",
      "Event ID:  000000006\n",
      "Event ID:  000000030\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7331a9d0dd6945dfa09e5d35e5c87daa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14ab5bba647c4db694c375c74d69cec0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  000000000\n",
      "Event ID:  000000024\n",
      "Event ID:  000000039\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f7d8d60617a4aab8090530fd73c5a81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "268ec801e4f046fbae924ccacfe86d41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3d5044fd2bc4dbaba81f94b330f7680",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e4594856c314081a666af10213905d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de95ba64256d43e59c1090bb7dc2a78e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5db9a899623a497da6bd49168691d4dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5413094ea1b84b31ac508ff661214e3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83fcc64723e0466fab7524a2a089959c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=34106), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e5c69c88eef4dabbeb547e265ad045b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=29911), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skooch/.local/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py:547: UserWarning: Multiprocessing-backed parallel loops cannot be nested, setting n_jobs=1\n",
      "  **self._backend_args)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "584d5528434e4ebbafa9bbb77cb6cbfb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=36830), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skooch/.local/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py:547: UserWarning: Multiprocessing-backed parallel loops cannot be nested, setting n_jobs=1\n",
      "  **self._backend_args)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  000000016\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "074fdb17c553439e90262712a62f3181",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  000000034\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "262a316b1ef94b5ea8342fbffb0e09f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skooch/.local/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py:547: UserWarning: Multiprocessing-backed parallel loops cannot be nested, setting n_jobs=1\n",
      "  **self._backend_args)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  000000019\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c129ab49e9144bca711835be6b7d316",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5528aac6a15b401a886d92fbad526fce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=39022), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "submission = create_test_submissions(start=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13741466\n"
     ]
    }
   ],
   "source": [
    "event_ids = [ i for i in range(0,125) ]\n",
    "submissions = []\n",
    "for i,event_id in enumerate(event_ids):\n",
    "    submission  = pd.read_csv('./%09d.agg.csv.gz'%event_id, compression='gzip')\n",
    "    submissions.append(submission)\n",
    "\n",
    "# Create submission file\n",
    "submission = pd.concat(submissions, axis=0)\n",
    "submission.to_csv('20180701_agg_e_35.csv.gz', index=False, compression='gzip')\n",
    "print(len(submission))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "70612062632493a78bec5bd5c69c5d4d523b8b83"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "c0bcf488d3b05ba63ad0b15b13db62b445ddbe3b"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_uuid": "c5c3e83193bf36741e4607a2da9a3b0777b18322"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "122321c5503e451c87c10017bd11adb1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-c13d2d89a5ab>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;31m# Track pattern recognition\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mClusterer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.0037\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;31m# Prepare submission for an event\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-2b4c95d03e3e>\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, hits)\u001b[0m\n\u001b[1;32m    104\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclusters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    107\u001b[0m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_preprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m         cl = hdbscan.HDBSCAN(min_samples=1,min_cluster_size=7,\n",
      "\u001b[0;32m<ipython-input-8-2b4c95d03e3e>\u001b[0m in \u001b[0;36m_init\u001b[0;34m(self, dfh)\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[0;31m#             clusters=DBSCAN(eps=self.epsilon+ii*stepeps,min_samples=1,metric='euclidean', n_jobs=1).fit(dfs).labels_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m             \u001b[0mclusters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAgglomerativeClustering\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdfs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mii\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m                 \u001b[0mdfh\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m's1'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclusters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/cluster/hierarchical.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    740\u001b[0m             memory.cache(tree_builder)(X, connectivity,\n\u001b[1;32m    741\u001b[0m                                        \u001b[0mn_clusters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_clusters\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 742\u001b[0;31m                                        **kwargs)\n\u001b[0m\u001b[1;32m    743\u001b[0m         \u001b[0;31m# Cut the tree\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    744\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcompute_full_tree\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/externals/joblib/memory.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    360\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    361\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 362\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    363\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    364\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcall_and_shelve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/cluster/hierarchical.py\u001b[0m in \u001b[0;36mward_tree\u001b[0;34m(X, connectivity, n_clusters, return_distance)\u001b[0m\n\u001b[1;32m    181\u001b[0m                           \u001b[0;34m'for the specified number of clusters'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m                           stacklevel=2)\n\u001b[0;32m--> 183\u001b[0;31m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhierarchy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    184\u001b[0m         \u001b[0mchildren_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mintp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    185\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/scipy/cluster/hierarchy.py\u001b[0m in \u001b[0;36mward\u001b[0;34m(y)\u001b[0m\n\u001b[1;32m    480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m     \"\"\"\n\u001b[0;32m--> 482\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mlinkage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ward'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'euclidean'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/scipy/cluster/hierarchy.py\u001b[0m in \u001b[0;36mlinkage\u001b[0;34m(y, method, metric, optimal_ordering)\u001b[0m\n\u001b[1;32m    710\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"`y` must be 1 or 2 dimensional.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    711\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 712\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfinite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    713\u001b[0m         raise ValueError(\"The condensed distance matrix must contain only \"\n\u001b[1;32m    714\u001b[0m                          \"finite values.\")\n",
      "\u001b[0;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "dataset_submissions = []\n",
    "dataset_scores = []\n",
    "for event_id, hits, cells, particles, truth in load_dataset(path_to_train, skip=0, nevents=5):\n",
    "    # Track pattern recognition\n",
    "    model = Clusterer(eps=0.0037)\n",
    "    labels = model.predict(hits)\n",
    "\n",
    "    # Prepare submission for an event\n",
    "    one_submission = create_one_event_submission(event_id, hits, labels)\n",
    "    dataset_submissions.append(one_submission)\n",
    "\n",
    "    # Score for the event\n",
    "    score = score_event(truth, one_submission)\n",
    "    dataset_scores.append(score)\n",
    "\n",
    "    print(\"Score for event %d: %.8f\" % (event_id, score))\n",
    "print('Mean score: %.8f' % (np.mean(dataset_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict one hit\n",
    "model = Clusterer(eps=0.0040)\n",
    "labels = model.predict(hits)\n",
    "\n",
    "submission = create_one_event_submission(0, hits, labels)\n",
    "score = score_event(truth, submission)\n",
    "print(\"Your score: \", score)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}