0.2-tweets-sentiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "collection = \"tweets_fernandezfernandez\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "xILzsBEsKl-R"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "pd.set_option('display.max_rows', 999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = pd.read_csv('./csvs/%s_tweets.csv' % collection, encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/pablogcelayes/Projects/escucha-tec/notebooks/csvs\n"
     ]
    }
   ],
   "source": [
    "cd /Users/pablogcelayes/Projects/escucha-tec/notebooks/csvs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tweets_hayotrocamino_tweets.csv  tweets_hayotrocamino_users.csv\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "DQtQl17lpt0T"
   },
   "outputs": [],
   "source": [
    "tweets['created_at'] = pd.to_datetime(tweets['created_at'],\n",
    "                                      format=\"%a %b %d %H:%M:%S +0000 %Y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "223973"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agregar sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "VECTORS_PATH = \"/Users/pablogcelayes/WordVectors/vectors/\"\n",
    "\n",
    "from os.path import join"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/pablogcelayes/Projects/sentiment-spanish/tass2019\n"
     ]
    }
   ],
   "source": [
    "cd /Users/pablogcelayes/Projects/sentiment-spanish/tass2019/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/tass-2019/lib/python3.6/site-packages/tass2019-0.1-py3.6.egg/sentiment/tokenizer.py:11: UserWarning: TreeTagger not installed. Lemmatization not available.\n",
      "  warnings.warn('TreeTagger not installed. Lemmatization not available.')\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "Error reading file '/anaconda3/envs/tass-2019/lib/python3.6/site-packages/tass2019-0.1-py3.6.egg/public_dat/es/intertass_es_train.xml': failed to load external entity \"/anaconda3/envs/tass-2019/lib/python3.6/site-packages/tass2019-0.1-py3.6.egg/public_dat/es/intertass_es_train.xml\"",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-09b5fd8e617d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msentiment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclassifier\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSentimentClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m model = SentimentClassifier(**{\n\u001b[1;32m      3\u001b[0m     \u001b[0;34m'clf'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'maxent'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;34m'clf_params'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;34m'bow'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/envs/tass-2019/lib/python3.6/site-packages/tass2019-0.1-py3.6.egg/sentiment/classifier.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensemble\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRandomForestClassifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mExtraTreesClassifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAdaBoostClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msentiment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSentimentTokenizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msentiment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membeddings\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSentenceVectorizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mWeightedSentenceVectorizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0membeddings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTweetTokenizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/envs/tass-2019/lib/python3.6/site-packages/tass2019-0.1-py3.6.egg/sentiment/tokenizer.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0membeddings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTweetTokenizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msentiment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnegation\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mhandle_negations\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnegation_tokens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0msettings\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtree_tagger_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtree_tagger_params_path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msentiment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0memoji\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mEmojiDict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/envs/tass-2019/lib/python3.6/site-packages/tass2019-0.1-py3.6.egg/sentiment/settings.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mcorpora\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mlang\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'es'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'cr'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'pe'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'uy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m     \u001b[0mcorpora\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_train'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mInterTASSReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtass2019_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'{0}/intertass_{0}_train.xml'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m     \u001b[0mcorpora\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_dev'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mInterTASSReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtass2019_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'{0}/intertass_{0}_dev.xml'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0mcorpora\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_test'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mInterTASSReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtass2019_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'intertass_{0}_test.xml'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/envs/tass-2019/lib/python3.6/site-packages/tass2019-0.1-py3.6.egg/sentiment/tass.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, filename, res_filename)\u001b[0m\n\u001b[1;32m    156\u001b[0m         \u001b[0;31m# https://stackoverflow.com/questions/13046240/parseerror-not-well-formed-invalid-token-using-celementtree\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m         \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0metree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mXMLParser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecover\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 158\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mroot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0metree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetroot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtweets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32msrc/lxml/etree.pyx\u001b[0m in \u001b[0;36mlxml.etree.parse\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32msrc/lxml/parser.pxi\u001b[0m in \u001b[0;36mlxml.etree._parseDocument\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32msrc/lxml/parser.pxi\u001b[0m in \u001b[0;36mlxml.etree._parseDocumentFromURL\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32msrc/lxml/parser.pxi\u001b[0m in \u001b[0;36mlxml.etree._parseDocFromFile\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32msrc/lxml/parser.pxi\u001b[0m in \u001b[0;36mlxml.etree._BaseParser._parseDocFromFile\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32msrc/lxml/parser.pxi\u001b[0m in \u001b[0;36mlxml.etree._ParserContext._handleParseResultDoc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32msrc/lxml/parser.pxi\u001b[0m in \u001b[0;36mlxml.etree._handleParseResult\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32msrc/lxml/parser.pxi\u001b[0m in \u001b[0;36mlxml.etree._raiseParseError\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: Error reading file '/anaconda3/envs/tass-2019/lib/python3.6/site-packages/tass2019-0.1-py3.6.egg/public_dat/es/intertass_es_train.xml': failed to load external entity \"/anaconda3/envs/tass-2019/lib/python3.6/site-packages/tass2019-0.1-py3.6.egg/public_dat/es/intertass_es_train.xml\""
     ]
    }
   ],
   "source": [
    "from sentiment.classifier import SentimentClassifier\n",
    "model = SentimentClassifier(**{\n",
    "    'clf': 'maxent',\n",
    "    'clf_params': None,\n",
    "    'bow': True,\n",
    "    'bow_params': {'binary': True, 'ngram_range': (1, 5)},\n",
    "    'boc': True,\n",
    "    'boc_params': {'analyzer': 'char', 'binary': True, 'ngram_range': (1, 6)},\n",
    "    'emb': True,\n",
    "    'emb_params': {\n",
    "        'binarize': True,\n",
    "        'normalize': False, \n",
    "        'a': 0.1,\n",
    "#         'file': join(VECTORS_PATH, 'SENTIMENT_UBAv2_50_2.bin'),\n",
    "        'file': join(VECTORS_PATH, 'SENTIMENT_UBAv2_50_1.bin'),\n",
    "    }\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "1.0-análisis-tweets-mongo.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python [conda env:tass-2019]",
   "language": "python",
   "name": "conda-env-tass-2019-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}