Data Cleaning and Preprocessing Romney.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer,WordNetLemmatizer\n",
    "from wordsegment import segment\n",
    "from html import unescape\n",
    "import itertools\n",
    "import random\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xl_file = pd.ExcelFile(\"training-Obama-Romney-tweets-sampled.xlsx\", encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfs = {sheet_name: xl_file.parse('Romney',encoding='utf-8') \n",
    "          for sheet_name in xl_file.sheet_names}\n",
    "romneyData=dfs.get('Romney')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clearColumns(dataDF):\n",
    "    dataDF=dataDF.drop(dataDF.columns[[0, 1, 2, 5]], axis=1)\n",
    "    dataDF.columns = ['tweet', 'label']\n",
    "    dataDF = dataDF.ix[1:]\n",
    "    print(dataDF)\n",
    "    print(dataDF['label'].unique())\n",
    "    dataDF=dataDF[dataDF['label'].isin([0,1,-1,u'-1',u'1',u'0'])]\n",
    "    return dataDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                  tweet label\n",
      "1     Insidious!<e>Mitt Romney</e>'s Bain Helped Phi...    -1\n",
      "2     Senior <e>Romney</e> Advisor Claims <e>Obama</...     2\n",
      "3     .@WardBrenda @shortwave8669 @allanbourdius you...    -1\n",
      "4     <e>Mitt Romney</e> still doesn't <a>believe</a...    -1\n",
      "5     <e>Romney</e>'s <a>tax plan</a> deserves a 2nd...    -1\n",
      "6     Hope <e>Romney</e> debate prepped w/ the same ...     1\n",
      "7     Want to know how <e>Mitt Romney</e> is going t...    -1\n",
      "8     If <e>Romney</e> wins the <a>presidential elec...    -1\n",
      "9     Presidential debate round 2: <e>Romney</e> wan...     2\n",
      "10    Someone on the <e>mitt Romney</e> <a>Facebook ...  !!!!\n",
      "11    <e>Romney</e>'s <a>12 million jobs scam </a>re...    -1\n",
      "12    @LoreleiMission \"girl look at that body x 3 Th...  !!!!\n",
      "13    <e>Mitt #Romney</e> <a>said</a> that <a> catch...    -1\n",
      "14    Please be<e> Mitt Romney</e> “@HuffingtonPost:...     0\n",
      "15    <e>Romney</e> leads <e>Obama</e> on economy, j...     2\n",
      "16    #<e>Romney</e> is disappointed when states all...    -1\n",
      "17    If women that the Polls show are truly voting ...     0\n",
      "18    @politicswnation When it comes 2 his<a> $5 tri...    -1\n",
      "19    <e>Romney</e> can go eff himself n so can Obam...     2\n",
      "20    Tony Corleone, Tony Soprano &amp; Tony <e>\"Mit...  !!!!\n",
      "21    MT @LOLGOP | REMINDER:<e> Mitt Romney</e> isn'...    -1\n",
      "22    ppl know <e>Romney</e> is<a> a liar, he hides ...    -1\n",
      "23    @MelianDialogue @eimajuno \"I am not voting for...  !!!!\n",
      "24    Good luck<e> @MittRomney</e> tonight. I have a...     1\n",
      "25    <e>Mitt Romney </e> <a>is</a> seriously thick....    -1\n",
      "26    The debate tonight is going to be pointless we...     1\n",
      "27    Democrats Launch Ingenious New Site To Attack ...    -1\n",
      "28    Ha! Damn autocorrect...either that or <e>Mitt ...  !!!!\n",
      "29    <e>Mitt Romney</e> always <a>worshiped his fat...     0\n",
      "30    Hope the <a>debate</a> goes as well for <e>Rom...     1\n",
      "...                                                 ...   ...\n",
      "7765  CNN's John King: After Two Debates\"\" <e>Romney...     1\n",
      "7766  @brandibelle73 @jamesotto Team #<e>romney</e> ...     1\n",
      "7767   When Piers Morgan is saying that <e>Romney</e...     1\n",
      "7768             Obama <a>lol</a> <e>#Romneywinning</e>     1\n",
      "7769   Those who don't support <e>Romney</e> because...     1\n",
      "7770  No more liars in the White House  VOTE <e>Romn...     1\n",
      "7771  Excellent description: RT @Salon: <e>\"Mitt Rom...     1\n",
      "7772  \"@_JoseGoodTimesz: So on a <a>Trevecca survey<...     1\n",
      "7773   I want <e>Romney</e> to win just so he can pr...     1\n",
      "7774   Time for a <a>business mind</a>.  #<e>Romney<...     1\n",
      "7775  @Baker4lfe @barackobama @mittromney how many b...     1\n",
      "7776   Lmao ðŸ˜‚ðŸ˜‚â€œ@Rellzay: If y'all piss me of...     1\n",
      "7777  Love the way <e>Romney</e> <a>looks,</a> Presi...     1\n",
      "7778  And I'm about to put my <e>Romney</e>/Ryan <a>...     1\n",
      "7779   <e>Romney</e> <a>leads</a> by 1 in tight Flor...     1\n",
      "7780  @TLeech_87 its a tie, but I think <e>Obama</e>...     1\n",
      "7781   Why <e>Romney</e> won the <a>debate</a> big l...     1\n",
      "7782  Ross Perot Endorses <e>Romney</e> http://t.co/...     1\n",
      "7783   @orangeinah Still think I'm voting for <e>Rom...     1\n",
      "7784   The <a>Rasmussen tracking poll</a> shows <e>R...     1\n",
      "7785   PIERS MORGAN: Mitt <e>Romney</e> might just <...     1\n",
      "7786  @americanthink need more of this. I suspect mo...     1\n",
      "7787   Top<e> Romney </e>Adviser Calls Romneyâ€™s Ne...     1\n",
      "7788  I predict Mitt <e>Romney</e>  will win the <a>...     1\n",
      "7789  <e>Mitt Romney </e>fix this mess I know you ca...     1\n",
      "7790                         <e>romney</e> is gonna win     1\n",
      "7791   I just gained so much more respect of <e>Romn...     1\n",
      "7792  <e> Romney</e> up 17 in <a>CD</a> 6 is good ne...     1\n",
      "7793  Pray for another <e>Romney</e> big win in deba...     1\n",
      "7794  <e> Romney</e> up 17 in <a>CD</a> 6 is good ne...     1\n",
      "\n",
      "[7794 rows x 2 columns]\n",
      "[-1 2 1 '!!!!' 0 nan 'IR']\n"
     ]
    }
   ],
   "source": [
    "romneyDataDF = clearColumns(romneyData)\n",
    "#romneyData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# stopwords=['i','me','my','myself','we','our','ours','ourselves','you','your','yours','yourself','yourselves','he','him','his',\n",
    "#            'himself','she','her','hers','herself','it','its','itself','they','them','their','theirs','themselves','what',\n",
    "#            'which','who','whom','this','that','these','those','am','is','are','was','were','be','been','being','have','has',\n",
    "#            'had','having','do','does','did','doing','a','an','the','and','but','if','or','because','as','until','while','of',\n",
    "#            'at','by','for','with','about','against','between','into','through','during','before','after','above','below','to',\n",
    "#            'from','up','down','in','out','on','off','over','under','again','further','then','once','here','there','when',\n",
    "#            'where','why','how','all','any','both','each','few','more','most','other','some','such','only',\n",
    "#            'own','same','so','than','too','very','can','will','just','should','now','im']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cleanURLS(tweet):\n",
    "    tweet=re.sub('http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-f][0-9a-f]))+', '', tweet)\n",
    "    return tweet\n",
    "    \n",
    "def getText_fromHTML(tweet):\n",
    "    soup = BeautifulSoup(tweet)\n",
    "    tweet = soup.get_text()\n",
    "    return tweet\n",
    "\n",
    "def removeAppostophes(tweet):\n",
    "    appostophes = {\"s\": \"is\", \"re\": \"are\", \"em\": \"them\", \"ll\": \"will\", \"t\": \"it\", \"m\": \"am\", \"ve\": \"have\", \"d\": \"did\"}\n",
    "    words = []\n",
    "    [words.append(terms) for word in tweet.split() for terms in [appostophes[term] if term in appostophes else term for\n",
    "                                                                 term in word.split(\"'\")]]\n",
    "    tweet = \" \".join(words)\n",
    "    return tweet\n",
    "\n",
    "def splitAttachedwords(tweet):\n",
    "    tweet = \" \".join(segment(tweet))\n",
    "    return tweet\n",
    "\n",
    "def lemmatisation(tweet):\n",
    "    return \" \".join(WordNetLemmatizer().lemmatize(word) for word in tweet.split())\n",
    "\n",
    "def stemming(tweet):\n",
    "    return \" \".join(PorterStemmer().stem(word) for word in tweet.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cleanTweets(dataDF):\n",
    "    rowsToBeRemoved=[]\n",
    "    TAG_RE = re.compile(r'<[^>]+>')\n",
    "    for index, row in dataDF.iterrows():\n",
    "        tweet=row['tweet']\n",
    "        if isinstance(row['label'], str) == True:\n",
    "            row['label']=int(row['label'])\n",
    "        output=\"\"      \n",
    "        if isinstance(tweet, str) == False:\n",
    "            rowsToBeRemoved.append(index)\n",
    "            continue\n",
    "        tweet=tweet.lower()\n",
    "        \n",
    "        # remove the urls from tweet\n",
    "        tweet = cleanURLS(tweet)\n",
    "\n",
    "        # remove HTML tags\n",
    "        tweet = getText_fromHTML(tweet)\n",
    "        \n",
    "        # remove appostophes\n",
    "        tweet = removeAppostophes(tweet)\n",
    "        \n",
    "        # split attached words ex: goodboy = good boy\n",
    "        tweet = splitAttachedwords(tweet)\n",
    "        \n",
    "        for word in tweet.split(\" \"):\n",
    "            # removing usernames\n",
    "            if word.strip().startswith('@'):\n",
    "                word=\"\"\n",
    "            # removing hashtags\n",
    "            if word.strip().startswith('#'):\n",
    "                word=word[1:]\n",
    "            # strip punctuation\n",
    "            word=re.sub(r'[^\\w\\s]','',word)\n",
    "            if word != \"\" and word.isalpha():\n",
    "                output+=\" \"+word\n",
    "        if len(output)<1:\n",
    "            rowsToBeRemoved.append(index)\n",
    "        output = lemmatisation(output)\n",
    "        output = stemming(output)   \n",
    "        row['tweet']=output.strip()\n",
    "    return dataDF.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "romneyCleanedData=cleanTweets(romneyDataDF.copy(deep=True))\n",
    "# romneyCleanedData.dropna(axis=0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7790</th>\n",
       "      <td>romney is gonna win</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7791</th>\n",
       "      <td>i just gain so much more respect of romney</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7792</th>\n",
       "      <td>is good news add that to the</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7793</th>\n",
       "      <td>pray for anoth romney big win in debat today</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7794</th>\n",
       "      <td>is good news add that to the</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             tweet label\n",
       "7790                           romney is gonna win     1\n",
       "7791    i just gain so much more respect of romney     1\n",
       "7792                  is good news add that to the     1\n",
       "7793  pray for anoth romney big win in debat today     1\n",
       "7794                  is good news add that to the     1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "romneyCleanedData.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Shuffling the cleaned data\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from numpy.random import random_integers as ri\n",
    "\n",
    "final_data = romneyCleanedData.copy(deep=True)\n",
    "for i in range(0,500):\n",
    "    rand_int = ri(0,500)\n",
    "    X_train,X_test,y_train,y_test = train_test_split(final_data['tweet'],final_data['label'],test_size=0.5,random_state=rand_int)\n",
    "    train_dataDF = pd.concat([X_train,y_train],axis=1)\n",
    "    test_dataDF = pd.concat([X_test,y_test],axis=1)\n",
    "    final_data = pd.concat([train_dataDF,test_dataDF],axis=0)\n",
    "final_data.to_csv('romneyCleanedData_with_sampling_2.csv',sep=',',encoding='utf-8',header=['tweet','label'],index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}