news_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "<h1>NEWS HEADLINE CLASSIFICATION</h1>  \n",
    "\n",
    "<p>In this notebook we will try to classify news using <b>MULTINOMIAL NAIVE BAYES</b>. We will convert text into vector by using <b>Bag Of Words</b> model. We will use two different datasets from two differrent sources and merge them.Our objective is to find maximum accuracy possible</p> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "##Loading all the required libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB,GaussianNB\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "import time\n",
    "\n",
    "#library for regular expretion\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Loading the data which is from Kaggle\n",
    "#This dataset contains four news class -b for business, e- entertainment, -t for technology and science, -m for medical\n",
    "data = pd.read_csv(\"uci-news-aggregator.csv\",encoding='latin-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TITLE</th>\n",
       "      <th>CATEGORY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://domesticfuel.com/2014/05/02/oregon-inst...</td>\n",
       "      <td>Nature_Environment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://peakoil.com/generalideas/peak-oil-theor...</td>\n",
       "      <td>Nature_Environment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://theenergycollective.com/maxbaumhefner/3...</td>\n",
       "      <td>Nature_Environment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://dailykos.com/story/2014/05/02/1296329/-...</td>\n",
       "      <td>Nature_Environment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://theenergycollective.com/todayinenergy/3...</td>\n",
       "      <td>Nature_Environment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               TITLE            CATEGORY\n",
       "0  http://domesticfuel.com/2014/05/02/oregon-inst...  Nature_Environment\n",
       "1  http://peakoil.com/generalideas/peak-oil-theor...  Nature_Environment\n",
       "2  http://theenergycollective.com/maxbaumhefner/3...  Nature_Environment\n",
       "3  http://dailykos.com/story/2014/05/02/1296329/-...  Nature_Environment\n",
       "4  http://theenergycollective.com/todayinenergy/3...  Nature_Environment"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Loading another data which contain 5 different class of news\n",
    "#Nature_Environment , Politics, Lifestyle, Business_finance and Health\n",
    "data1 = pd.read_csv(\"ArticlesNewsSitesData_2382.csv\",sep=';')\n",
    "data1.columns = ['TITLE','CATEGORY']\n",
    "#\n",
    "data1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Extracting news haedline from urls \n",
    "for i in range(2382):\n",
    "    data1.TITLE[i] = re.sub(r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))', '', data1.TITLE[i])\n",
    "    data1.TITLE[i] = re.sub(r'(\\/[0-9]*\\/)*[0-9]*\\/', '', data1.TITLE[i])\n",
    "    data1.TITLE[i]=data1.TITLE[i].replace(\"-\",\" \")\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data1['TITLE'].replace('', np.nan, inplace=True)\n",
    "#data1.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TITLE</th>\n",
       "      <th>CATEGORY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2376</th>\n",
       "      <td>people watching.html</td>\n",
       "      <td>Lifestyle_leisure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2377</th>\n",
       "      <td>The Who s Roger Daltry Rocking for Philly Can...</td>\n",
       "      <td>Lifestyle_leisure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2379</th>\n",
       "      <td>stylehoroscope5e1a15f0 98e5 11e3 80ac 63a8ba7f...</td>\n",
       "      <td>Lifestyle_leisure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2380</th>\n",
       "      <td>dario franchitti interview indianapolis 500 ca...</td>\n",
       "      <td>Lifestyle_leisure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2381</th>\n",
       "      <td>flower power oils for loss and the stages of g...</td>\n",
       "      <td>Lifestyle_leisure</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  TITLE           CATEGORY\n",
       "2376                               people watching.html  Lifestyle_leisure\n",
       "2377   The Who s Roger Daltry Rocking for Philly Can...  Lifestyle_leisure\n",
       "2379  stylehoroscope5e1a15f0 98e5 11e3 80ac 63a8ba7f...  Lifestyle_leisure\n",
       "2380  dario franchitti interview indianapolis 500 ca...  Lifestyle_leisure\n",
       "2381  flower power oils for loss and the stages of g...  Lifestyle_leisure"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1.dropna(subset=['TITLE'], inplace=True)\n",
    "data1.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data = data[['TITLE','CATEGORY']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['b', 't', 'e', 'm', 'Nature_Environment', 'Politics',\n",
       "       'Science_Technology', 'Economy-Business_Finance', 'Health',\n",
       "       'Lifestyle_leisure'], dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Merging both datasets\n",
    "temp=[data,data1]\n",
    "news_data = pd.concat(temp)\n",
    "#Printing all different types of categories\n",
    "news_data.CATEGORY.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#news_data.groupby('CATEGORY').describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#converting category column into numeric target NUM_CATEGORY column\n",
    "news_data['NUM_CATEGORY']=news_data.CATEGORY.map({'b':0,'e':1,'m':2,'t':3,'Economy-Business_Finance':0,'Health':2,'Lifestyle_leisure':2,'Nature_Environment':4,'Politics':0,'Science_Technology':3})\n",
    "#news_data.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Splitting dataset into 60% training set and 40% test set\n",
    "x_train, x_test, y_train, y_test = train_test_split(news_data.TITLE, news_data.NUM_CATEGORY, random_state=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Here we convert our dataset into a Bag Of Word model using a Bigram model\n",
    "#start = time.clock()\n",
    "\n",
    "vect = CountVectorizer(ngram_range=(2,2))\n",
    "#converting traning features into numeric vector\n",
    "X_train = vect.fit_transform(x_train)\n",
    "#converting training labels into numeric vector\n",
    "X_test = vect.transform(x_test)\n",
    "\n",
    "#print (time.clock()-start)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Training and Predicting the data\n",
    "\n",
    "#start = time.clock()\n",
    "\n",
    "mnb = MultinomialNB(alpha=0.2)\n",
    "\n",
    "mnb.fit(X_train,y_train)\n",
    "\n",
    "result= mnb.predict(X_test)\n",
    "\n",
    "#print (time.clock()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.92421958893012568"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Printing accuracy of the our model\n",
    "accuracy_score(result,y_test)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#This function return the class of the input news\n",
    "def predict_news(news):\n",
    "    test = vect.transform(news)\n",
    "    pred= mnb.predict(test)\n",
    "    if pred  == 0:\n",
    "        return 'Business and Politics'\n",
    "    elif pred == 1:\n",
    "        return 'Entertainment'\n",
    "    elif pred == 2:\n",
    "        return 'Health and Lifestyle'\n",
    "    elif pred == 3:\n",
    "        return 'Science and Technology'\n",
    "    elif pred == 4:\n",
    "        return 'Environment'\n",
    "    \n",
    "    \n"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py35]",
   "language": "python",
   "name": "conda-env-py35-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}