outliers_parallel_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "e081740e-8169-4481-b1df-f5dd5488314f",
    "_uuid": "0bee86255243664f24e4bcf48af2228a3100a8b7"
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import timeit\n",
    "\n",
    "from trackml.dataset import load_event, load_dataset\n",
    "from trackml.score import score_event\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import hdbscan\n",
    "from scipy import stats\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.cluster import DBSCAN\n",
    "\n",
    "def create_one_event_submission(event_id, hits, labels):\n",
    "    sub_data = np.column_stack(([event_id]*len(hits), hits.hit_id.values, labels))\n",
    "    submission = pd.DataFrame(data=sub_data, columns=[\"event_id\", \"hit_id\", \"track_id\"]).astype(int)\n",
    "    return submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "572fcbb6-8c7b-4a09-8916-8ec76689130f",
    "_uuid": "63414de98667e95f60407c9155899a25a321cffc"
   },
   "outputs": [],
   "source": [
    "# Change this according to your directory preferred setting\n",
    "path_to_train = \"./data/train_1\"\n",
    "\n",
    "# This event is in Train_1\n",
    "event_prefix = \"event000001000\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "e06d1ed7-5091-4d67-abb4-5984b137e2e6",
    "_uuid": "c2f70ae63abffcc09a534bb17fb89df8ffddb722",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class Clusterer(object):\n",
    "    def __init__(self,rz_scales=[0.65, 0.965, 1.528], eps=0.0035):                        \n",
    "        self.rz_scales=rz_scales\n",
    "        self.epsilon = eps\n",
    "        \n",
    "    def _eliminate_outliers(self,labels,M):\n",
    "        norms=np.zeros((len(labels)),np.float32)\n",
    "        indices=np.zeros((len(labels)),np.float32)\n",
    "        for i, cluster in tqdm(enumerate(labels),total=len(labels)):\n",
    "            if cluster == 0:\n",
    "                continue\n",
    "            index = np.argwhere(self.clusters==cluster)\n",
    "            index = np.reshape(index,(index.shape[0]))\n",
    "            indices[i] = len(index)\n",
    "            x = M[index]\n",
    "            norms[i] = self._test_quadric(x)\n",
    "        threshold1 = np.percentile(norms,90)*5\n",
    "        threshold2 = 25\n",
    "        threshold3 = 6\n",
    "        for i, cluster in enumerate(labels):\n",
    "            if norms[i] > threshold1 or indices[i] > threshold2 or indices[i] < threshold3:\n",
    "                self.clusters[self.clusters==cluster]=0   \n",
    "    \n",
    "    def _test_quadric(self,x):\n",
    "        if x.size == 0 or len(x.shape)<2:\n",
    "            return 0\n",
    "        Z = np.zeros((x.shape[0],10), np.float32)\n",
    "        Z[:,0] = x[:,0]**2\n",
    "        Z[:,1] = 2*x[:,0]*x[:,1]\n",
    "        Z[:,2] = 2*x[:,0]*x[:,2]\n",
    "        Z[:,3] = 2*x[:,0]\n",
    "        Z[:,4] = x[:,1]**2\n",
    "        Z[:,5] = 2*x[:,1]*x[:,2]\n",
    "        Z[:,6] = 2*x[:,1]\n",
    "        Z[:,7] = x[:,2]**2\n",
    "        Z[:,8] = 2*x[:,2]\n",
    "        Z[:,9] = 1\n",
    "        v, s, t = np.linalg.svd(Z,full_matrices=False)        \n",
    "        smallest_index = np.argmin(np.array(s))\n",
    "        T = np.array(t)\n",
    "        T = T[smallest_index,:]        \n",
    "        norm = np.linalg.norm(np.dot(Z,T), ord=2)**2\n",
    "        return norm\n",
    "\n",
    "    def _preprocess(self, hits):\n",
    "        \n",
    "        x = hits.x.values\n",
    "        y = hits.y.values\n",
    "        z = hits.z.values\n",
    "\n",
    "        r = np.sqrt(x**2 + y**2 + z**2)\n",
    "        hits['x2'] = x/r\n",
    "        hits['y2'] = y/r\n",
    "\n",
    "        r = np.sqrt(x**2 + y**2)\n",
    "        hits['z2'] = z/r\n",
    "\n",
    "        ss = StandardScaler()\n",
    "        X = ss.fit_transform(hits[['x2', 'y2', 'z2']].values)\n",
    "        for i, rz_scale in enumerate(self.rz_scales):\n",
    "            X[:,i] = X[:,i] * rz_scale\n",
    "       \n",
    "        return X\n",
    "    \n",
    "    def _init(self,dfh):\n",
    "        dfh['r'] = np.sqrt(dfh['x'].values**2+dfh['y'].values**2+dfh['z'].values**2)\n",
    "        dfh['rt'] = np.sqrt(dfh['x'].values**2+dfh['y'].values**2)\n",
    "        dfh['a0'] = np.arctan2(dfh['y'].values,dfh['x'].values)\n",
    "        dfh['z1'] = dfh['z'].values/dfh['rt'].values\n",
    "        dfh['x2'] = 1/dfh['z1'].values\n",
    "        dz0 = -0.00070\n",
    "        stepdz = 0.00001\n",
    "        stepeps = 0.000005\n",
    "        mm = 1\n",
    "        for ii in tqdm(range(100)):\n",
    "            mm = mm*(-1)\n",
    "            dz = mm*(dz0+ii*stepdz)\n",
    "            dfh['a1'] = dfh['a0'].values+dz*dfh['z'].values*np.sign(dfh['z'].values)\n",
    "            dfh['sina1'] = np.sin(dfh['a1'].values)\n",
    "            dfh['cosa1'] = np.cos(dfh['a1'].values)\n",
    "            dfh['x1'] = dfh['a1'].values/dfh['z1'].values\n",
    "            ss = StandardScaler()\n",
    "            dfs = ss.fit_transform(dfh[['sina1','cosa1','z1','x1','x2']].values)\n",
    "            cx = np.array([1, 1, 0.75, 0.5, 0.5])\n",
    "            dfs = np.multiply(dfs, cx)\n",
    "\n",
    "            clusters=DBSCAN(eps=self.epsilon+ii*stepeps,min_samples=1,metric='euclidean',n_jobs=-1).fit(dfs).labels_            \n",
    "            if ii==0:\n",
    "                dfh['s1'] = clusters\n",
    "                dfh['N1'] = dfh.groupby('s1')['s1'].transform('count')\n",
    "            else:\n",
    "                dfh['s2'] = clusters\n",
    "                dfh['N2'] = dfh.groupby('s2')['s2'].transform('count')\n",
    "                maxs1 = dfh['s1'].max()\n",
    "                cond = np.where((dfh['N2'].values>dfh['N1'].values) & (dfh['N2'].values<20))\n",
    "                s1 = dfh['s1'].values\n",
    "                s1[cond] = dfh['s2'].values[cond]+maxs1\n",
    "                dfh['s1'] = s1\n",
    "                dfh['s1'] = dfh['s1'].astype('int64')\n",
    "                dfh['N1'] = dfh.groupby('s1')['s1'].transform('count')\n",
    "        return dfh['s1'].values    \n",
    "    \n",
    "    def predict(self, hits):    \n",
    "        self.clusters = self._init(hits) \n",
    "        X = self._preprocess(hits) \n",
    "        cl = hdbscan.HDBSCAN(min_samples=1,min_cluster_size=7,\n",
    "                             metric='braycurtis',cluster_selection_method='leaf',algorithm='best', leaf_size=50)\n",
    "        labels = np.unique(self.clusters)\n",
    "        self._eliminate_outliers(labels,X)\n",
    "        n_labels = 0\n",
    "        while n_labels < len(labels):\n",
    "            n_labels = len(labels)            \n",
    "            max_len = np.max(self.clusters)\n",
    "            mask = self.clusters == 0\n",
    "            self.clusters[mask] = cl.fit_predict(X[mask])+max_len\n",
    "        return self.clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "c5c3e83193bf36741e4607a2da9a3b0777b18322"
   },
   "outputs": [],
   "source": [
    "dataset_submissions = []\n",
    "dataset_scores = []\n",
    "for event_id, hits, cells, particles, truth in load_dataset(path_to_train, skip=0, nevents=5):\n",
    "    # Track pattern recognition\n",
    "    model = Clusterer(eps=0.0040)\n",
    "    labels = model.predict(hits)\n",
    "\n",
    "    # Prepare submission for an event\n",
    "    one_submission = create_one_event_submission(event_id, hits, labels)\n",
    "    dataset_submissions.append(one_submission)\n",
    "\n",
    "    # Score for the event\n",
    "    score = score_event(truth, one_submission)\n",
    "    dataset_scores.append(score)\n",
    "\n",
    "    print(\"Score for event %d: %.8f\" % (event_id, score))\n",
    "print('Mean score: %.8f' % (np.mean(dataset_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "70612062632493a78bec5bd5c69c5d4d523b8b83"
   },
   "outputs": [],
   "source": [
    "model = Clusterer(eps=0.0040)\n",
    "labels = model.predict(hits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "c0bcf488d3b05ba63ad0b15b13db62b445ddbe3b"
   },
   "outputs": [],
   "source": [
    "submission = create_one_event_submission(0, hits, labels)\n",
    "score = score_event(truth, submission)\n",
    "print(\"Your score: \", score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_cell_guid": "2b20cf0c-2754-48dd-ab1e-4f489c2aa05c",
    "_uuid": "7f8de52b9022581bf10aa813d2db005b842f0be7",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path_to_test = \"./data/test\"\n",
    "start = 0\n",
    "\n",
    "def create_test_submissions(path_to_test = \"./data/test\", start=0):\n",
    "    test_dataset_submissions = []\n",
    "    \n",
    "    for event_id, hits, cells in load_dataset(path_to_test, parts=['hits', 'cells']):\n",
    "        if event_id >= start:\n",
    "            print('Event ID: ', event_id)\n",
    "\n",
    "            # Track pattern recognition \n",
    "            model = Clusterer(eps=0.0040)\n",
    "            labels = model.predict(hits)\n",
    "\n",
    "            # Prepare submission for an event\n",
    "            one_submission = create_one_event_submission(event_id, hits, labels)\n",
    "            one_submission.to_csv('./%09d.helix.csv.gz'%event_id, index=False, compression='gzip')\n",
    "            test_dataset_submissions.append(one_submission)\n",
    "\n",
    "    # Create submission file\n",
    "    submission = pd.concat(test_dataset_submissions, axis=0)\n",
    "\n",
    "    return submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event ID:  0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bd60153f65d449ea9dbb81cc0bc318a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2523856a3337406c89efa0bfd1e5568c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=36766), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Event ID:  1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eaa20f3854a74bd7917e94f8097d99b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68f0cd8820d1492b8a6ecd86bd6bd3c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=37815), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Event ID:  2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1212973bb104469186c0045356bb26b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92a53bb8692f43d7a3cc402670dffaf7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=36884), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Event ID:  3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f64427c6ab0c404a97125aa4d1971f5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-56d4f34d7156>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_test_submissions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-7-f108d3f7ec56>\u001b[0m in \u001b[0;36mcreate_test_submissions\u001b[0;34m(path_to_test, start)\u001b[0m\n\u001b[1;32m     12\u001b[0m                 \u001b[0;31m# Track pattern recognition\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m                 \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mClusterer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.0040\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m                 \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m                 \u001b[0;31m# Prepare submission for an event\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-f53b1348409b>\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, hits)\u001b[0m\n\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclusters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_preprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         cl = hdbscan.HDBSCAN(min_samples=1,min_cluster_size=7,\n",
      "\u001b[0;32m<ipython-input-6-f53b1348409b>\u001b[0m in \u001b[0;36m_init\u001b[0;34m(self, dfh)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0mdfs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultiply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdfs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mclusters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDBSCAN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mii\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mstepeps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmin_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmetric\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'euclidean'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdfs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mii\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m                 \u001b[0mdfh\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m's1'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclusters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/cluster/dbscan_.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    282\u001b[0m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'csr'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m         clust = dbscan(X, sample_weight=sample_weight,\n\u001b[0;32m--> 284\u001b[0;31m                        **self.get_params())\n\u001b[0m\u001b[1;32m    285\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore_sample_indices_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclust\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore_sample_indices_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/cluster/dbscan_.py\u001b[0m in \u001b[0;36mdbscan\u001b[0;34m(X, eps, min_samples, metric, metric_params, algorithm, leaf_size, p, sample_weight, n_jobs)\u001b[0m\n\u001b[1;32m    143\u001b[0m         \u001b[0;31m# This has worst case O(n^2) memory complexity\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m         neighborhoods = neighbors_model.radius_neighbors(X, eps,\n\u001b[0;32m--> 145\u001b[0;31m                                                          return_distance=False)\n\u001b[0m\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msample_weight\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/neighbors/base.py\u001b[0m in \u001b[0;36mradius_neighbors\u001b[0;34m(self, X, radius, return_distance)\u001b[0m\n\u001b[1;32m    623\u001b[0m                     \"or set algorithm='brute'\" % self._fit_method)\n\u001b[1;32m    624\u001b[0m             results = self._tree.query_radius(X, radius,\n\u001b[0;32m--> 625\u001b[0;31m                                               return_distance=return_distance)\n\u001b[0m\u001b[1;32m    626\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mreturn_distance\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    627\u001b[0m                 \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "_ = create_test_submissions(start=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "event_ids = [ i for i in range(0,125) ]\n",
    "submissions = []\n",
    "for i,event_id in enumerate(event_ids):\n",
    "    submission  = pd.read_csv('./%09d.helix.csv.gz'%event_id, compression='gzip')\n",
    "    submissions.append(submission)\n",
    "\n",
    "# Create submission file\n",
    "submission = pd.concat(submissions, axis=0)\n",
    "submission.to_csv('20180701_dbscan_e_345.csv.gz', index=False, compression='gzip')\n",
    "print(len(submission))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}