themes_viz_generator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Thematic Clustering of Fact-Checked Stories\n",
    "\n",
    "This notebook clusters the headlines of fact-checking stories in the Tattle archive using an algorithm called GSDMM. The output file is used to generate a visualisation on the Tattle website.\n",
    "\n",
    "### Process\n",
    "\n",
    "\n",
    "1. Getting the data from MongoDB \n",
    "2. Text cleaning (removing noise, English / non-English headlines separation using regex)\n",
    "3. Translating non-English headlines \n",
    "4. Pre-processing all the headlines (tokenizing, stop word removal, lemmatizing, creating bigrams)\n",
    "5. Text transformation: creating a corpus of vectors\n",
    "6. Building the GSDMM model\n",
    "7. Adding cluster labels to headlines\n",
    "8. Interactive model visualisation with pyLDAvis\n",
    "9. Adding article links and total count to output file\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Getting the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing libraries\n",
    "import os\n",
    "import requests\n",
    "import time\n",
    "from time import sleep\n",
    "from random import uniform\n",
    "import datetime\n",
    "from datetime import date, timezone\n",
    "import csv\n",
    "from pymongo import MongoClient\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "import os\n",
    "from os import environ\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pprint import pprint\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import spacy\n",
    "import gensim\n",
    "import gensim.corpora as corpora\n",
    "from gensim.corpora import Dictionary\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.models import ldamodel\n",
    "from gensim.models import CoherenceModel \n",
    "import re\n",
    "from langdetect import detect\n",
    "from gensim.models.phrases import Phrases, Phraser\n",
    "from nltk import FreqDist\n",
    "from nltk.corpus import RegexpTokenizer as regextoken\n",
    "import matplotlib\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "import logging\n",
    "#logging.basicConfig(filename='lda_model.log', format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "# import googletrans\n",
    "# from googletrans import Translator\n",
    "from google_trans_new import google_translator  \n",
    "import pyLDAvis\n",
    "import pyLDAvis.gensim\n",
    "from pyLDAvis import PreparedData\n",
    "from gensim import similarities\n",
    "import nbconvert\n",
    "from gsdmm import MovieGroupProcess\n",
    "import pickle\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get data from factchecking sites MongoDB\n",
    "def initialize_mongo():\n",
    "    #mongo_url = \"mongodb+srv://\"+os.environ.get(\"FACTCHECK_DEV_DB_USERNAME\")+\":\"+os.environ.get(\"FACTCHECK_DEV_DB_PASSWORD\")+\"@tattle-data-fkpmg.mongodb.net/test?retryWrites=true&w=majority&ssl=true&tlsallowinvalidcertificates=false\"   \n",
    "    cli = MongoClient(mongo_url)\n",
    "    #db = cli[os.environ.get(\"FACTCHECK_DEV_DB_NAME\")]\n",
    "    #coll = db[os.environ.get(\"FACTCHECK_DEV_DB_COLLECTION\")]\n",
    "    db = cli[\"factcheck_sites_dev\"]\n",
    "    coll = db[\"stories\"]\n",
    "    if coll.count_documents({}) > 0:\n",
    "        return coll \n",
    "    else:\n",
    "        print(\"Error accessing Mongo collection\")\n",
    "        sys.exit()\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymongo/uri_parser.py:328: DeprecationWarning: Option 'ssl_cert_reqs' is deprecated, use 'tlsAllowInvalidCertificates' instead.\n",
      "  options = _handle_option_deprecations(options)\n"
     ]
    }
   ],
   "source": [
    "coll = initialize_mongo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14999"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll.count_documents({})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weekly_data(coll):\n",
    "    pipeline = [\n",
    "        {\"$project\":{\"date_accessed\":\"$date_accessed\", \"date_updated\":\"$date_updated\", 'postID': \"$postID\",'postURL': \"$postURL\",\n",
    "                     \"headline\": \"$headline\", \"docs\": \"$docs\", \"author\": \"$author\", \"domain\": \"$domain\",\n",
    "                     \"date\": {\"$dateFromString\": {\"dateString\": \"$date_updated\"}}}},\n",
    "        {\"$match\": {\"date\": {\"$gte\":datetime.datetime(2020, 8, 30, 0, 0), \"$lt\": datetime.datetime(2020, 9, 30, 0, 0)}}}\n",
    "        #{\"$match\": {\"date\": {\"$gte\":datetime.datetime(2019, 1, 1, 0, 0), \"$lt\": datetime.datetime(2020, 1, 1, 0, 0)}}}\n",
    "    ]\n",
    "    \n",
    "    docs = coll.aggregate(pipeline)\n",
    "    return docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "153\n"
     ]
    }
   ],
   "source": [
    "c=0\n",
    "result=[]\n",
    "docs = get_weekly_data(coll)\n",
    "for doc in docs:\n",
    "    result.append(doc)\n",
    "    c+=1\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('FCDBJulyAugust2019.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>date_accessed</th>\n",
       "      <th>date_updated</th>\n",
       "      <th>postID</th>\n",
       "      <th>postURL</th>\n",
       "      <th>headline</th>\n",
       "      <th>docs</th>\n",
       "      <th>author</th>\n",
       "      <th>domain</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5f89ae9c1c9a4d25843f93c3</td>\n",
       "      <td>October 16, 2020</td>\n",
       "      <td>September 21, 2020</td>\n",
       "      <td>3a9fe1e718b4470d9d9f7b23262b2aec</td>\n",
       "      <td>https://www.altnews.in/italy-defeated-covid-19...</td>\n",
       "      <td>Italy defeated COVID-19? Misinformation-riddle...</td>\n",
       "      <td>[{'doc_id': 'c542a9546530401f9c63e87859e1b442'...</td>\n",
       "      <td>{'name': 'Dr Sharfaroz Satani', 'link': 'https...</td>\n",
       "      <td>altnews.in</td>\n",
       "      <td>2020-09-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5f89ae9e1c9a4d25843f93c5</td>\n",
       "      <td>October 16, 2020</td>\n",
       "      <td>September 27, 2020</td>\n",
       "      <td>9efdb44103144107b44d2e9bba1c8af8</td>\n",
       "      <td>https://www.altnews.in/ayush-kwath-or-kadha-ca...</td>\n",
       "      <td>AYUSH Kwath or Kadha cannot ‘boost’ immunity t...</td>\n",
       "      <td>[{'doc_id': 'cc9ec77a0c704cf783ed6c009e425119'...</td>\n",
       "      <td>{'name': 'Dr. Sumaiya Shaikh', 'link': 'https:...</td>\n",
       "      <td>altnews.in</td>\n",
       "      <td>2020-09-27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>602ac324e654233d4c456935</td>\n",
       "      <td>February 15, 2021</td>\n",
       "      <td>September 18, 2020</td>\n",
       "      <td>a4872a5476c84b34983e26d2aff92f4d</td>\n",
       "      <td>https://bangla.boomlive.in/fact-check/fake-new...</td>\n",
       "      <td>আজানের সময় মন্দিরে মাইক নয়, ভুয়ো পোস্ট জানা...</td>\n",
       "      <td>[{'doc_id': '53b66ce6b64141e7a63e4963a89ab8aa'...</td>\n",
       "      <td>{'name': 'Suhash Bhattacharjee', 'link': '/aut...</td>\n",
       "      <td>bangla.boomlive.in</td>\n",
       "      <td>2020-09-18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        _id      date_accessed        date_updated  \\\n",
       "0  5f89ae9c1c9a4d25843f93c3   October 16, 2020  September 21, 2020   \n",
       "1  5f89ae9e1c9a4d25843f93c5   October 16, 2020  September 27, 2020   \n",
       "2  602ac324e654233d4c456935  February 15, 2021  September 18, 2020   \n",
       "\n",
       "                             postID  \\\n",
       "0  3a9fe1e718b4470d9d9f7b23262b2aec   \n",
       "1  9efdb44103144107b44d2e9bba1c8af8   \n",
       "2  a4872a5476c84b34983e26d2aff92f4d   \n",
       "\n",
       "                                             postURL  \\\n",
       "0  https://www.altnews.in/italy-defeated-covid-19...   \n",
       "1  https://www.altnews.in/ayush-kwath-or-kadha-ca...   \n",
       "2  https://bangla.boomlive.in/fact-check/fake-new...   \n",
       "\n",
       "                                            headline  \\\n",
       "0  Italy defeated COVID-19? Misinformation-riddle...   \n",
       "1  AYUSH Kwath or Kadha cannot ‘boost’ immunity t...   \n",
       "2  আজানের সময় মন্দিরে মাইক নয়, ভুয়ো পোস্ট জানা...   \n",
       "\n",
       "                                                docs  \\\n",
       "0  [{'doc_id': 'c542a9546530401f9c63e87859e1b442'...   \n",
       "1  [{'doc_id': 'cc9ec77a0c704cf783ed6c009e425119'...   \n",
       "2  [{'doc_id': '53b66ce6b64141e7a63e4963a89ab8aa'...   \n",
       "\n",
       "                                              author              domain  \\\n",
       "0  {'name': 'Dr Sharfaroz Satani', 'link': 'https...          altnews.in   \n",
       "1  {'name': 'Dr. Sumaiya Shaikh', 'link': 'https:...          altnews.in   \n",
       "2  {'name': 'Suhash Bhattacharjee', 'link': '/aut...  bangla.boomlive.in   \n",
       "\n",
       "        date  \n",
       "0 2020-09-21  \n",
       "1 2020-09-27  \n",
       "2 2020-09-18  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "clean_df = df.drop_duplicates(subset=[\"postURL\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "151"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clean_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Italy defeated COVID-19? Misinformation-riddle...\n",
       "1    AYUSH Kwath or Kadha cannot ‘boost’ immunity t...\n",
       "2    আজানের সময় মন্দিরে মাইক নয়, ভুয়ো পোস্ট জানা...\n",
       "Name: headline, dtype: object"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Snapshot of headlines\n",
    "clean_df[\"headline\"][0:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Text cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/core/frame.py:3607: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._set_item(key, value)\n"
     ]
    }
   ],
   "source": [
    "# Defining a function to remove accented characters in the headlines  \n",
    "def remove_accents(sentence):\n",
    "    return \", \".join(simple_preprocess(str(sentence), deacc=True, max_len=100))\n",
    "clean_df[\"unaccented\"] = clean_df[\"headline\"].map(remove_accents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Detecting non-English headlines using regex\n",
    "def detect_lang(text):\n",
    "    pat = re.compile(\"[^\\x00-\\x7F]\") # matches non-English characters\n",
    "    if pat.search(text):\n",
    "        return 0\n",
    "    else:\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_df[\"is_english\"] = clean_df[\"unaccented\"].map(detect_lang)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Translating non-English headlines\n",
    "\n",
    "Googletrans is a free library that sends translation requests to the Google Translate API. \n",
    "Random time delays between requests are advised, else Google may (and probably will) block your ip address.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.42 s, sys: 1.09 s, total: 4.51 s\n",
      "Wall time: 6min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# Translating non-English headlines using googletrans library\n",
    "translator = google_translator()\n",
    "clean_df[\"english_headline\"] = \"\"\n",
    "for i, row in clean_df.iterrows():\n",
    "    if row[\"is_english\"] == 0:\n",
    "        if coll.count_documents({\"_id\": row[\"_id\"], \"english_headline\": {\"$exists\": True}}) > 0:\n",
    "            clean_df.at[i, \"english_headline\"] = coll.find_one({\"_id\": row[\"_id\"]})[\"english_headline\"]\n",
    "        else:\n",
    "            clean_df.at[i, \"english_headline\"] = translator.translate(row[\"headline\"], lang_tgt='en')\n",
    "            time.sleep(uniform(3,5))\n",
    "    else:\n",
    "        clean_df.at[i, \"english_headline\"] = row[\"headline\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n",
      "<ipython-input-18-3ba1a8bee80d>:3: DeprecationWarning: update is deprecated. Use replace_one, update_one or update_many instead.\n",
      "  coll.update(\n"
     ]
    }
   ],
   "source": [
    "# Store translated headlines in Mongo\n",
    "for i, row in clean_df.iterrows():\n",
    "    coll.update(\n",
    "    {\"_id\": row[\"_id\"]},\n",
    "    {\"$set\": {\"english_headline\": row[\"english_headline\"]}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>headline</th>\n",
       "      <th>english_headline</th>\n",
       "      <th>is_english</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>আজানের সময় মন্দিরে মাইক নয়, ভুয়ো পোস্ট জানা...</td>\n",
       "      <td>During Azan, Mike is not Mike in the temple, M...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>'গালওয়ানের প্রমাণ' হিসেবে আজতক, টাইমস নাও দেখা...</td>\n",
       "      <td>Azatak, Times, Times, as a proof of 'Galwan' s...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>সম্পর্কহীন ছবি সহ ম্যাডোনার হিন্দু ধর্ম সম্পর্...</td>\n",
       "      <td>Viral comments about Madonon Hindu religion wi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            headline  \\\n",
       "2  আজানের সময় মন্দিরে মাইক নয়, ভুয়ো পোস্ট জানা...   \n",
       "3  'গালওয়ানের প্রমাণ' হিসেবে আজতক, টাইমস নাও দেখা...   \n",
       "6  সম্পর্কহীন ছবি সহ ম্যাডোনার হিন্দু ধর্ম সম্পর্...   \n",
       "\n",
       "                                    english_headline  is_english  \n",
       "2  During Azan, Mike is not Mike in the temple, M...           0  \n",
       "3  Azatak, Times, Times, as a proof of 'Galwan' s...           0  \n",
       "6  Viral comments about Madonon Hindu religion wi...           0  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_df[clean_df[\"is_english\"]==0][[\"headline\", \"english_headline\",\"is_english\"]][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>headline</th>\n",
       "      <th>english_headline</th>\n",
       "      <th>is_english</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Doctored Video Falsely Shared To Claim Hindu M...</td>\n",
       "      <td>Doctored Video Falsely Shared To Claim Hindu M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Did UNSC Remove Taliban From Terror List Under...</td>\n",
       "      <td>Did UNSC Remove Taliban From Terror List Under...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Old Video Of Afghans Crossing Pak Border Share...</td>\n",
       "      <td>Old Video Of Afghans Crossing Pak Border Share...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            headline  \\\n",
       "0  Doctored Video Falsely Shared To Claim Hindu M...   \n",
       "1  Did UNSC Remove Taliban From Terror List Under...   \n",
       "2  Old Video Of Afghans Crossing Pak Border Share...   \n",
       "\n",
       "                                    english_headline  is_english  \n",
       "0  Doctored Video Falsely Shared To Claim Hindu M...           1  \n",
       "1  Did UNSC Remove Taliban From Terror List Under...           1  \n",
       "2  Old Video Of Afghans Crossing Pak Border Share...           1  "
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_df[clean_df[\"is_english\"]==1][[\"headline\", \"english_headline\",\"is_english\"]][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_df.to_csv(\"clean_df.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Text preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "141"
      ]
     },
     "execution_count": 384,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clean_df[\"english_headline\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tokenizing the headlines\n",
    "def sent_to_words(sentences):\n",
    "    for sentence in sentences:\n",
    "        yield (simple_preprocess(str(sentence), deacc = True))\n",
    "        \n",
    "all_tokens = list(sent_to_words(clean_df[\"english_headline\"]))\n",
    "# Creating stop words list\n",
    "stop_words = stopwords.words(\"english\")\n",
    "# Adding domain words \n",
    "stop_words.extend([\"fake\", \"fact\", \"check\", \"checked\", \"factcheck\", \"news\", \"false\", \n",
    "                   \"falsely\", \"true\", \"truth\", \"viral\", \"video\", \"image\", \"picture\", \n",
    "                   \"photo\", \"claim\", \"claiming\", \"share\", \"clip\", \"misleading\",\"recent\", \"old\",\n",
    "                  \"india\", \"post\", \"medium\", \"go\"])\n",
    "# Stop word removal\n",
    "data_stopped = [[word for word in doc if word not in stop_words] for doc in all_tokens]\n",
    "# Creating bigrams\n",
    "bigram = gensim.models.Phrases(data_stopped, min_count=2)\n",
    "for idx in range(len(data_stopped)):\n",
    "    for token in bigram[data_stopped[idx]]:\n",
    "        if '_' in token:\n",
    "            # If token is bigram, add it to document\n",
    "            data_stopped[idx].append(token)\n",
    "            \n",
    "data_with_bigrams = data_stopped\n",
    "# Lemmatizing i.e. reducing words to their root form\n",
    "# Including only nouns as this improves both topic interpretability and coherence scores\n",
    "def lemmatization(docs, allowed_postags=[\"NOUN\", \"PROPN\", \"VERB\", \"ADJ\", \"ADV\"]):\n",
    "    nlp = spacy.load(\"en_core_web_sm\")\n",
    "    docs_out = []\n",
    "    for sent in docs:\n",
    "        doc = nlp(\" \".join(sent))\n",
    "        docs_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags]) \n",
    "    return docs_out\n",
    "\n",
    "data_lemmatized = lemmatization(data_with_bigrams, allowed_postags=[\"NOUN\", \"PROPN\", \"VERB\", \"ADJ\", \"ADV\"])\n",
    "\n",
    "# Removing any stopwords created because of lemmatization\n",
    "data_cleaned = [[word for word in doc if word not in stop_words] for doc in data_lemmatized]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Text transformation: creating a corpus\n",
    "\n",
    "Topic modelling with the Gensim library involves documents, corpus, vectors and bag of words. These are explained here - https://radimrehurek.com/gensim/auto_examples/core/run_core_concepts.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "141"
      ]
     },
     "execution_count": 386,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_cleaned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique tokens: 552\n"
     ]
    }
   ],
   "source": [
    "# Creating a dictionary\n",
    "id2word = corpora.Dictionary(data_cleaned)\n",
    "# Creating a document-term matrix\n",
    "print('Number of unique tokens: %d' % len(id2word))\n",
    "#id2word.filter_extremes(no_below = 20)\n",
    "# Creating a document-term matrix\n",
    "corpus = [id2word.doc2bow(doc) for doc in data_cleaned]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['doctored', 'hindu', 'man', 'lynch', 'delhi']"
      ]
     },
     "execution_count": 388,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_cleaned[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GSDMM clustering "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 422,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(9)   #41,5\n",
    "mgp = MovieGroupProcess(K=5, alpha=0.1, beta=0.1, n_iters=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In stage 0: transferred 85 clusters with 5 clusters populated\n",
      "In stage 1: transferred 56 clusters with 5 clusters populated\n",
      "In stage 2: transferred 41 clusters with 5 clusters populated\n",
      "In stage 3: transferred 39 clusters with 5 clusters populated\n",
      "In stage 4: transferred 42 clusters with 5 clusters populated\n",
      "In stage 5: transferred 43 clusters with 5 clusters populated\n",
      "In stage 6: transferred 42 clusters with 5 clusters populated\n",
      "In stage 7: transferred 43 clusters with 5 clusters populated\n",
      "In stage 8: transferred 37 clusters with 5 clusters populated\n",
      "In stage 9: transferred 34 clusters with 5 clusters populated\n",
      "In stage 10: transferred 34 clusters with 5 clusters populated\n",
      "In stage 11: transferred 34 clusters with 5 clusters populated\n",
      "In stage 12: transferred 34 clusters with 5 clusters populated\n",
      "In stage 13: transferred 38 clusters with 5 clusters populated\n",
      "In stage 14: transferred 35 clusters with 5 clusters populated\n",
      "In stage 15: transferred 29 clusters with 5 clusters populated\n",
      "In stage 16: transferred 36 clusters with 5 clusters populated\n",
      "In stage 17: transferred 37 clusters with 5 clusters populated\n",
      "In stage 18: transferred 35 clusters with 5 clusters populated\n",
      "In stage 19: transferred 37 clusters with 5 clusters populated\n",
      "In stage 20: transferred 33 clusters with 5 clusters populated\n",
      "In stage 21: transferred 33 clusters with 5 clusters populated\n",
      "In stage 22: transferred 25 clusters with 5 clusters populated\n",
      "In stage 23: transferred 31 clusters with 5 clusters populated\n",
      "In stage 24: transferred 38 clusters with 5 clusters populated\n",
      "In stage 25: transferred 35 clusters with 5 clusters populated\n",
      "In stage 26: transferred 34 clusters with 5 clusters populated\n",
      "In stage 27: transferred 28 clusters with 5 clusters populated\n",
      "In stage 28: transferred 30 clusters with 5 clusters populated\n",
      "In stage 29: transferred 36 clusters with 5 clusters populated\n",
      "In stage 30: transferred 34 clusters with 5 clusters populated\n",
      "In stage 31: transferred 39 clusters with 5 clusters populated\n",
      "In stage 32: transferred 37 clusters with 5 clusters populated\n",
      "In stage 33: transferred 34 clusters with 5 clusters populated\n",
      "In stage 34: transferred 26 clusters with 5 clusters populated\n",
      "In stage 35: transferred 31 clusters with 5 clusters populated\n",
      "In stage 36: transferred 30 clusters with 5 clusters populated\n",
      "In stage 37: transferred 32 clusters with 5 clusters populated\n",
      "In stage 38: transferred 34 clusters with 5 clusters populated\n",
      "In stage 39: transferred 27 clusters with 5 clusters populated\n",
      "In stage 40: transferred 28 clusters with 5 clusters populated\n",
      "In stage 41: transferred 27 clusters with 5 clusters populated\n",
      "In stage 42: transferred 29 clusters with 5 clusters populated\n",
      "In stage 43: transferred 35 clusters with 5 clusters populated\n",
      "In stage 44: transferred 32 clusters with 5 clusters populated\n",
      "In stage 45: transferred 33 clusters with 5 clusters populated\n",
      "In stage 46: transferred 28 clusters with 5 clusters populated\n",
      "In stage 47: transferred 30 clusters with 5 clusters populated\n",
      "In stage 48: transferred 27 clusters with 5 clusters populated\n",
      "In stage 49: transferred 30 clusters with 5 clusters populated\n"
     ]
    }
   ],
   "source": [
    "vocab = set(x for doc in data_cleaned for x in doc)\n",
    "y = mgp.fit(data_cleaned,len(vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cluster labelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_df[\"tokens\"]=data_cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assign cluster label to each article headline\n",
    "clean_df[\"cluster\"] = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 426,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>english_headline</th>\n",
       "      <th>cluster</th>\n",
       "      <th>postURL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Doctored Video Falsely Shared To Claim Hindu M...</td>\n",
       "      <td>4</td>\n",
       "      <td>https://www.boomlive.in/fact-check/fact-check/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Did UNSC Remove Taliban From Terror List Under...</td>\n",
       "      <td>4</td>\n",
       "      <td>https://www.boomlive.in/fact-check/fact-check/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Old Video Of Afghans Crossing Pak Border Share...</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.boomlive.in/fact-check/fact-check/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>No, MP Government Did Not Raze A Slum Over 'Pa...</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.boomlive.in/fact-check/fact-check/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Shocking Assault Video From Chhattisgarh False...</td>\n",
       "      <td>2</td>\n",
       "      <td>https://www.boomlive.in/fact-check/fast-check/...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    english_headline  cluster  \\\n",
       "0  Doctored Video Falsely Shared To Claim Hindu M...        4   \n",
       "1  Did UNSC Remove Taliban From Terror List Under...        4   \n",
       "2  Old Video Of Afghans Crossing Pak Border Share...        0   \n",
       "3  No, MP Government Did Not Raze A Slum Over 'Pa...        0   \n",
       "4  Shocking Assault Video From Chhattisgarh False...        2   \n",
       "\n",
       "                                             postURL  \n",
       "0  https://www.boomlive.in/fact-check/fact-check/...  \n",
       "1  https://www.boomlive.in/fact-check/fact-check/...  \n",
       "2  https://www.boomlive.in/fact-check/fact-check/...  \n",
       "3  https://www.boomlive.in/fact-check/fact-check/...  \n",
       "4  https://www.boomlive.in/fact-check/fast-check/...  "
      ]
     },
     "execution_count": 426,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_df.head(5)[[\"english_headline\", \"cluster\", \"postURL\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 427,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    37\n",
       "2    37\n",
       "3    26\n",
       "0    23\n",
       "1    18\n",
       "Name: cluster, dtype: int64"
      ]
     },
     "execution_count": 427,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_df[\"cluster\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 428,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "https://www.boomlive.in/fact-check/fact-check/old-video-afghan-corona-pandemic-afghanistan-pakistan-border-viral-false-claim-14522\n",
      "https://www.boomlive.in/fact-check/fact-check/no-mp-government-did-not-raze-a-slum-over-pakistan-zindabad-slogans-14503\n",
      "https://www.boomlive.in/fact-check/fact-check/fake-news-viral-video-mock-drill-bank-robbery-ahmednagar-police-maharashtra-factcheck-14543\n",
      "https://www.boomlive.in/fact-check/fact-check/fir-against-coca-cola-for-penis-shaped-bottles-satire-post-viral-14493\n",
      "https://hindi.boomlive.in/fact-check/fact-check/top-five-fake-news-from-last-week-viral-video-14569\n",
      "https://hindi.boomlive.in/fact-check/international/usa-president-joe-biden-sleeping-israeli-prime-minister-naftali-bennett-meeting-fake-news-14516\n",
      "https://bangla.boomlive.in/fact-check/fact-check/old-video-afghan-corona-pandemic-afghanistan-pakistan-border-viral-false-claim-14572\n",
      "https://bangla.boomlive.in/fact-check/fact-check/fact-check-justice-dalveer-bhandari-india-icj-chief-justice-uk-71-years-183-votes-14494\n",
      "https://bangla.boomlive.in/fact-check/fact-check/photoshopped-image-of-nusrat-jahan-ruhi-with-her-baby-ishaan-shared-on-social-media-14521\n",
      "https://bangla.boomlive.in/fact-check/fact-check/photo-of-injured-injured-man-stitches-viral-farmers-protest-lathi-charge-haryana-imran-pratapgarhi-fake-news-14515\n",
      "https://newsmeter.in/fact-check/fact-check/no-50-off-on-hyderabad-traffic-challans-viral-msg-on-lok-adalat-is-false-682846\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-image-of-dustbins-with-pakistan-murdabad-stickers-shared-with-misleading-claim-682742\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-is-this-fingerprint-shaped-village-located-in-germany-682629\n",
      "https://digiteye.in/free-baleno-car-offer-in-a-contest-by-maruti-suzuki-on-40th-anniversary-making-rounds-fact-check/\n",
      "https://factly.in/telugu-central-government-is-not-selling-off-assets-worth-6-lakh-crores-through-the-proposed-national-asset-monetisation-pipeline/\n",
      "https://factly.in/telugu-no-truth-in-the-claim-that-ambedkar-was-elected-to-rajya-sabha-with-the-help-of-jana-sang-after-losing-1952-lok-sabha-elections/\n",
      "https://factly.in/telugu-illegal-structures-demolished-by-ujjain-government-officials-falsely-linked-to-people-who-raised-pro-pakistan-slogans/\n",
      "https://factly.in/nilkamal-furniture-company-had-not-affixed-these-pakistan-murdabad-sticker-on-their-dustbins/\n",
      "https://factly.in/telugu-hyderabad-traffic-police-are-not-providing-opportunity-to-pay-pending-traffic-challans-with-50-discount/\n",
      "https://factly.in/no-hyderabad-traffic-police-is-not-organising-any-traffic-lok-adalat-during-oct-4-7-2021/\n",
      "https://www.vishvasnews.com/politics/fact-check-post-claiming-rs-20000-crore-worth-of-investments-of-pf-and-pension-fund-may-lose-is-misleading/\n",
      "https://www.vishvasnews.com/politics/fact-check-pakistan-afghanistan-border-an-old-video-now-goes-viral-with-fake-claims/\n",
      "https://www.vishvasnews.com/politics/fact-check-viral-message-in-the-name-of-rajasthan-government-regarding-mosque-madrasa-is-fake/\n",
      "1\n",
      "https://www.boomlive.in/fact-check/fact-check/video-of-delhi-politician-criticising-congress-viral-as-fictitious-anil-upadhyay-14513\n",
      "https://hindi.boomlive.in/fact-check/fact-check/aaj-tak-viral-tweet-screenshot-bjp-apologises-to-gandhi-family-fake-news-fact-checkfact-check-14556\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-does-mumbai-really-have-28-billionaires-682683\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-did-congress-mla-criticize-sonia-and-rahul-gandhi-682702\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-aaj-taks-tweet-on-bjp-apologizing-to-gandhi-family-is-fake-682800\n",
      "https://newsmeter.in/fact-check/telangana/bandi-sanjay-blames-kcr-for-postponement-of-huzurabad-by-election-682899\n",
      "https://factly.in/telugu-supreme-court-has-not-issued-notices-to-15-media-channels-asking-to-apologise-to-the-tablighi-jamaat/\n",
      "https://factly.in/video-of-indian-soldiers-becoming-unconscious-due-to-heat-stroke-is-shared-with-an-anti-vaccine-narrative/\n",
      "https://factly.in/telugu-this-cheque-of-nirav-modi-donating-rs-98-crore-to-the-congress-party-is-fake/\n",
      "https://factly.in/telugu-telangana-high-court-has-directed-the-state-govt-not-to-open-only-those-schools-with-residential-hostels/\n",
      "https://factly.in/telugu-reliance-alloted-money-only-to-refurbish-this-existing-pathway-of-stairs-to-tirumala/\n",
      "https://factly.in/aajtak-tweet-saying-bjp-apologised-to-gandhi-family-in-a-court-is-digitally-morphed/\n",
      "https://factly.in/queen-elizabeth-hasnt-ordered-cristiano-ronaldo-autographed-jerseys-from-manchester-united/\n",
      "https://factly.in/telugu-morphed-survey-results-shared-to-indicate-rahul-gandhi-to-be-preferred-pm-candidate-over-modi/\n",
      "https://factly.in/telugu-ap-high-court-judge-was-not-arrested-on-suspicion-of-having-links-with-maoists/\n",
      "https://www.vishvasnews.com/politics/quick-fact-check-the-person-speaking-against-the-congress-in-the-viral-video-is-not-a-congress-mla/\n",
      "https://www.vishvasnews.com/politics/fact-check-this-tweet-in-the-name-of-aaj-tak-is-edited-and-fake/\n",
      "https://www.vishvasnews.com/politics/fact-check-karni-senas-president-video-viral-name-of-bjp-mla/\n",
      "2\n",
      "https://www.boomlive.in/fact-check/fast-check/assault-video-of-minor-girl-by-pastor-from-chhattisgarh-falsely-revived-as-dps-school-rajbag-valsad-gujarat-14507\n",
      "https://www.boomlive.in/fact-check/coronavirus-outbreak/tanya-cures-nature-5-crore-compensation-for-mandatory-forced-covid-19-vaccination-india-awaken-india-movement-14537\n",
      "https://www.boomlive.in/fact-check/fact-check/photo-of-children-reacting-to-airstrikes-in-gaza-falsely-shared-as-kabul-14562\n",
      "https://www.boomlive.in/fact-check/fact-check/ratan-tata-alcohol-aadhaar-card-subsidies-food-quote-fake-fact-check-14563\n",
      "https://www.boomlive.in/fact-check/fact-check/fact-check-sushil-kajal-farmers-protest-haryana-karnal-viral-image-different-person-14527\n",
      "https://www.boomlive.in/fact-check/fact-check/photo-of-injured-injured-man-stitches-viral-farmers-protest-lathi-charge-haryana-imran-pratapgarhi-fake-news-14495\n",
      "https://www.boomlive.in/fact-check/fact-check/blood-on-road-farmers-protest-old-image-militant-attack-cisf-jawans-srinagar-14486\n",
      "https://hindi.boomlive.in/fact-check/fact-check/siddharth-shukla-heart-attack-death-viral-video-fake-news-fact-check-14549\n",
      "https://hindi.boomlive.in/fact-check/fact-check/2016-photo-of-interfaith-couple-marriage-kapil-mishra-sister-viral-fact-check-14529\n",
      "https://hindi.boomlive.in/fact-check/fact-check/narendra-modi-and-imran-khan-meal-dinner-viral-photo-fake-news-14536\n",
      "https://hindi.boomlive.in/fact-check/fact-check/photo-of-injured-man-stiches-viral-as-karnal-kisan-lathi-charge-imran-pratapgarhi-fake-news-fact-check-14496\n",
      "https://hindi.boomlive.in/fact-check/fact-check/rajasthan-ashok-gehlot-madrasa-masjid-viral-message-ipc-427-fake-news-14554\n",
      "https://bangla.boomlive.in/fact-check/fact-check/fir-against-coca-cola-for-penis-shaped-bottles-satire-post-viral-14534\n",
      "https://bangla.boomlive.in/fact-check/fact-check/taliban-takeover-2015-image-of-security-check-in-yemen-shared-as-afghanistan-airport-14483\n",
      "https://bangla.boomlive.in/fact-check/fast-check/cropped-newspaper-dainik-jugasankha-headline-of-mamata-banerjee-quote-on-hinduism-revived-14505\n",
      "https://bangla.boomlive.in/fact-check/fact-check/blood-on-road-farmers-protest-old-image-militant-attack-cisf-jawans-srinagar-14504\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-photos-of-us-soldiers-killed-in-2017-aircraft-crash-shared-as-images-of-kabul-attack-victims-682651\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-photo-of-women-students-in-heavily-damaged-classroom-is-from-gaza-not-afghanistan-682680\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-viral-picture-of-cnn-journalist-surrounded-by-gunmen-is-from-syria-not-afghanistan-682674\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-did-taliban-kill-tolo-news-journalist-in-afghanistan-682623\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-old-image-of-militant-attack-in-srinagar-passed-off-as-crackdown-on-farmers-682653\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-did-ratan-tata-wanted-liquor-and-aadhaar-linked-no-682878\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-her-name-is-sadhvi-saraswati-and-she-did-not-kill-anyone-682658\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-old-image-of-protests-passed-off-as-aftermath-of-kabul-blasts-682620\n",
      "https://factly.in/old-photo-from-yemen-falsely-shared-as-visuals-of-security-check-at-an-airport-in-afghanistan/\n",
      "https://factly.in/old-exorcism-video-from-chhattisgarh-falsely-shared-as-visuals-of-dps-rajbagh-teacher-assaulting-a-young-girl/\n",
      "https://factly.in/while-there-is-no-tax-on-printed-school-books-there-is-tax-on-its-printing-and-supply/\n",
      "https://factly.in/old-video-from-gujarat-shared-as-visuals-of-a-dead-christian-family-in-taliban-controlled-afghanistan/\n",
      "https://factly.in/old-and-unrelated-videos-are-passed-off-as-those-of-recent-us-drone-strikes-in-kabul/\n",
      "https://factly.in/photo-of-an-injured-cow-vigilante-shared-as-a-farmers-injured-head-in-a-lathicharge-at-farmers-protest/\n",
      "https://www.vishvasnews.com/politics/fact-check-an-old-video-of-suicide-from-gujarat-is-falsely-shared-as-taliban-killing-the-christians-and-those-who-supported-the-previous-government-in-afghanistan/\n",
      "https://www.vishvasnews.com/politics/fact-check-no-this-image-of-a-female-journalist-is-not-related-with-the-taliban-but-from-syria/\n",
      "https://www.vishvasnews.com/politics/quick-fact-check-kapil-mishras-sister-is-not-married-to-a-muslim-boy-viral-post-is-fake/\n",
      "https://www.vishvasnews.com/politics/quick-fact-check-post-claiming-taxation-on-school-books-is-fake-school-books-are-gst-exempted/\n",
      "https://www.vishvasnews.com/politics/fact-check-ratan-tata-did-not-asked-to-remove-government-subsidy-to-liquor-buyers/?itm_source=homepage&itm_medium=mbl_s1&itm_campaign=editorpick\n",
      "https://www.vishvasnews.com/politics/top-factcheck-stories-top-fact-check-stories-of-the-weeks-on-afghan-crisis/?itm_source=homepage&itm_medium=mbl_s3&itm_campaign=editorpick\n",
      "https://www.vishvasnews.com/world/fact-check-this-picture-of-the-damaged-school-is-of-gaza-not-afghanistan/\n",
      "3\n",
      "https://www.boomlive.in/fact-check/fact-check/fake-news-joe-biden-falls-asleep-naftali-bennett-israeli-prime-minister-meeting-factcheck-14506\n",
      "https://hindi.boomlive.in/fact-check/fact-check/zee-hindustan-video-panjshir-minor-fighter-girl-against-taliban-afghanistan-fake-news-14557\n",
      "https://bangla.boomlive.in/fact-check/fact-check/gas-cylinder-store-gutted-in-fire-in-sa-pa-vietnam-falsely-shared-as-malaysia-14555\n",
      "https://bangla.boomlive.in/fact-check/fact-check/impeachment-time-this-magazine-cover-with-joe-biden-kamala-harris-is-fake-14487\n",
      "https://bangla.boomlive.in/fact-check/fact-check/fake-news-ujjain-pakistan-zindabad-slogan-ram-navami-gulbarga-karnataka-14482\n",
      "https://bangla.boomlive.in/fact-check/fact-check/14-year-teen-nasa-black-holes-god-space-research-false-information-14502\n",
      "https://bangla.boomlive.in/fact-check/fact-check/fake-news-joe-biden-falls-asleep-naftali-bennett-israeli-prime-minister-meeting-factcheck-14573\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-did-upper-caste-men-strip-dalit-woman-for-entering-temple-682757\n",
      "https://newsmeter.in/fact-check/fact-check/flowers-in-viral-image-are-not-mahameru-pushpam-682740\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-hindus-took-out-janmashtami-procession-in-kashmir-but-not-for-the-first-time-682704\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-viral-video-of-bride-slapping-groom-is-from-comedy-skit-682841\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-can-consumption-durian-before-and-after-covid-19-vaccination-prove-fatal-682676\n",
      "https://newsmeter.in/fact-check/nation/scr-records-51-increase-in-freight-loading-in-2021-682898\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-birds-in-viral-picture-are-not-malaysian-frogmouth-682782\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-two-images-of-destroyed-buildings-shared-with-false-claim-682799\n",
      "https://newsmeter.in/fact-check/hyderabad/ghmc-receives-22-excess-rainfall-this-monsoon-682900\n",
      "https://digiteye.in/janmashtami-celebrated-in-kashmir-in-2021-first-time-after-32-years-fact-check/\n",
      "https://factly.in/video-of-a-street-food-vendor-adding-his-urine-to-a-vessel-is-shared-with-a-false-communal-narrative/\n",
      "https://factly.in/telugu-comedy-skit-%e2%80%8cvideo-shared-as-the-bride-trashing-groom-for-chewing-gutka/\n",
      "https://factly.in/a-clipping-from-joe-bidens-meeting-with-prime-minister-of-israel-is-shared-with-false-context/\n",
      "https://factly.in/this-image-of-terraced-hills-landscape-is-from-vietnam-not-himachal-pradesh/\n",
      "https://factly.in/telugu-this-photo-is-not-related-to-the-incident-where-mothers-skeleton-was-found-by-her-son-returning-from-america/\n",
      "https://factly.in/telugu-congress-party-did-not-administer-such-few-polio-vaccines-when-in-power/\n",
      "https://factly.in/telugu-a-photo-taken-in-the-philippines-during-2020-typhoon-is-morphed-and-shared-as-situation-in-delhi/\n",
      "https://www.vishvasnews.com/viral/fact-check-this-is-a-digital-artwork-not-real-birds/\n",
      "https://www.vishvasnews.com/politics/fact-check-years-old-incident-of-dalit-woman-misbehaved-in-bihar-is-being-shared-as-recent-with-the-misleading-claim/\n",
      "4\n",
      "https://www.boomlive.in/fact-check/fact-check/fact-check-fake-news-muslims-attacked-hindu-man-in-delhi-viral-video-fake-news-fact-check-14551\n",
      "https://www.boomlive.in/fact-check/fact-check/taliban-terror-organisation-list-unsc-india-fact-check-14541\n",
      "https://www.boomlive.in/fact-check/fact-check/fake-news-viral-video-man-hanging-taliban-united-states-helicopter-afghanistan-kandahar-fact-check-14519\n",
      "https://www.boomlive.in/fact-check/fact-check/old-video-of-man-collapsing-shared-as-last-moments-of-sidharth-shukla-14530\n",
      "https://www.boomlive.in/fact-check/fast-check/1938-bks-iyengar-performing-yoga-revived-as-rare-video-of-pm-narendra-modi-14575\n",
      "https://www.boomlive.in/fact-check/fact-check/fake-news-aaj-tak-graphic-morphed-screenshot-afghanistan-taliban-united-states-narendra-modi-special-plan-factcheck-14564\n",
      "https://www.boomlive.in/fact-check/fact-check/fact-check-fake-news-zee-hindustan-taliban-afghanistan-panjshir-little-girl-firing-gun-machine-gun-14550\n",
      "https://www.boomlive.in/fact-check/fact-check/fake-news-pakistan-india-muslim-man-electricity-stealing-taliban-14567\n",
      "https://www.boomlive.in/fact-check/fact-check/2020-libyas-aircraft-video-aired-by-zee-24-ghanta-as-taliban-failed-to-fly-helicopter-14574\n",
      "https://hindi.boomlive.in/fact-check/fact-check/narendra-modi-nathuram-godse-savarkar-photo-viral-image-fact-check-fake-news-14576\n",
      "https://hindi.boomlive.in/fact-check/fact-check/muslim-man-stealing-electricity-and-threatening-to-kill-viral-video-bijli-chori-fake-news-14565\n",
      "https://bangla.boomlive.in/fact-check/fact-check/fake-news-viral-video-man-hanging-taliban-united-states-helicopter-afghanistan-kandahar-fact-check-14526\n",
      "https://bangla.boomlive.in/fact-check/fact-check/2020-libyas-aircraft-video-aired-by-zee-24-ghanta-as-taliban-failed-to-fly-helicopter-14552\n",
      "https://bangla.boomlive.in/fact-check/fact-check/weekly-wrap-top-5-fake-news-taliban-helicopter-video-fake-claim-urdu-flex-misleading-claim-14558\n",
      "https://bangla.boomlive.in/fact-check/fact-check/old-video-of-man-collapsing-shared-as-last-moments-of-sidharth-shukla-14553\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-video-of-mob-vandalizing-hindu-temple-is-from-pakistan-not-afghanistan-682681\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-aaj-tak-graphic-on-modis-special-plan-for-taliban-is-morphed-682897\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-image-from-balika-vadhu-shared-as-final-rites-of-late-actor-sidharth-shukla-682842\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-indian-media-gets-it-wrong-video-does-not-show-taliban-hanging-man-from-chopper-682758\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-image-of-muslim-boy-with-morphed-text-on-placard-goes-viral-682895\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-year-old-video-of-siddharth-shukla-being-shared-as-his-last-video-682837\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-man-collapsing-on-staircase-passed-off-as-last-moments-of-sidharth-shukla-682778\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-did-pakistani-taliban-threaten-india-in-a-video-post-682607\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-video-does-not-show-pandit-bhimsen-joshis-grandson-singing-682877\n",
      "https://newsmeter.in/fact-check/fact-check/fact-check-unsc-did-not-remove-taliban-from-terror-list-under-indias-presidency-682861\n",
      "https://digiteye.in/video-of-a-pakistani-man-defending-power-theft-passed-off-as-indian-muslim-fact-check/\n",
      "https://digiteye.in/old-video-of-pakistan-talibani-threatening-india-modi-reappears-as-recent-one-fact-check/\n",
      "https://factly.in/this-video-shows-a-muslim-mob-vandalising-a-hindu-temple-in-pakistan-not-afghanistan/\n",
      "https://factly.in/telugu-satyapal-singh-introduced-a-private-members-bill-in-2019-to-free-hindu-temples-from-government-control/\n",
      "https://factly.in/an-old-video-is-shared-as-pakistani-taliban-threatening-india-after-recent-taliban-takeover-of-afghanistan/\n",
      "https://factly.in/this-video-does-not-show-taliban-hanging-a-man-from-a-black-hawk-helicopter/\n",
      "https://factly.in/2020-video-from-libya-shared-as-taliban-trying-to-fly-black-hawk-helicopter/\n",
      "https://factly.in/divine-words-can-defeat-diabetes-says-research-not-exactly/\n",
      "https://www.vishvasnews.com/politics/fact-check-pm-modis-edited-photo-with-imran-khan-went-viral-again-in-the-name-of-up-elections/?itm_source=homepage&itm_medium=mbl_s2&itm_campaign=editorpick\n",
      "https://www.vishvasnews.com/politics/fact-check-an-old-video-of-a-pakistani-politician-is-being-shared-as-pakistan-taliban-threatening-to-india-after-taking-over-of-kabul-by-taliban/\n",
      "https://www.vishvasnews.com/viral/fact-check-balika-vadhu-serial-scene-of-siddhart-shukla-viral-as-his-last-rite/?itm_source=homepage&itm_medium=mbl_s4&itm_campaign=editorpick\n",
      "https://www.vishvasnews.com/viral/fact-check-this-is-not-the-last-video-of-sidharth-shukla-before-his-death-old-video-going-viral-with-false-claim/\n"
     ]
    }
   ],
   "source": [
    "# Create dictionary of clusters and headlines\n",
    "headlines_dict= {}\n",
    "c=1\n",
    "for i in clean_df.groupby(by=\"cluster\"):\n",
    "    print(i[0])\n",
    "    pairs_list = []\n",
    "    df = i[1][[\"headline\", \"postURL\", \"tokens\"]]\n",
    "#     print(df[\"tokens\"])\n",
    "#     print(\"...........\")\n",
    "    for idx, row in df.iterrows():\n",
    "        print(row[\"postURL\"])\n",
    "        pairs = {}\n",
    "        pairs[\"url\"] = row[\"postURL\"]\n",
    "        pairs[\"headline\"] = row[\"headline\"]\n",
    "        pairs_list.append(pairs)\n",
    "    headlines_dict[c] = pairs_list\n",
    "    c+=1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualise MGP model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 429,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pyLDAvis/_prepare.py:228: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only\n",
      "  default_term_info  = pd.DataFrame({'saliency': saliency, 'Term': vocab, \\\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el96947434267844708335610\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el96947434267844708335610_data = {\"mdsDat\": {\"x\": [0.2398951437716594, -0.16059970823710215, 0.23113778384626138, -0.3338659275307337, 0.023432708149915542], \"y\": [0.3361971981535235, -0.12886453938609765, -0.1872752542501586, 0.16118532320043402, -0.18124272771770125], \"topics\": [1, 2, 3, 4, 5], \"cluster\": [1, 1, 1, 1, 1], \"Freq\": [17.551607015758705, 16.305596788787962, 23.54086703243633, 16.43573884467161, 26.166190318345393]}, \"tinfo\": {\"Term\": [\"taliban\", \"pakistan\", \"farmer\", \"shukla\", \"congress\", \"gandhi\", \"man\", \"traffic\", \"last\", \"court\", \"happen\", \"uzjin\", \"oct\", \"jan\", \"round\", \"photograph\", \"suzuki\", \"zindabad\", \"official\", \"ambedkar\", \"traffic\", \"lok\", \"asset\", \"hyderabad\", \"border\", \"adalat\", \"hyderabad_traffic\", \"afghans\", \"sabha\", \"wound\", \"murdabad\", \"shape\", \"crore\", \"pakistan\", \"government\", \"sticker\", \"police\", \"cristiano\", \"speak\", \"election\", \"apologise\", \"aajtak\", \"lawyer\", \"anil\", \"reliance\", \"path\", \"manchester\", \"gandhi\", \"congress\", \"bjp\", \"court\", \"apologize\", \"tweet\", \"gandhi_family\", \"newsletter\", \"mla\", \"cost\", \"rahul\", \"order\", \"family\", \"high\", \"aaj\", \"tak\", \"gujarat\", \"cow\", \"alcohol\", \"vaccinate\", \"us\", \"syria\", \"shared_visuals\", \"damage\", \"christian\", \"classroom\", \"farmer\", \"kabul\", \"kill\", \"tax\", \"protest\", \"school_books\", \"ratan\", \"srinagar\", \"book\", \"journalist\", \"tata\", \"visual\", \"school\", \"afghanistan\", \"taliban\", \"bird\", \"dalit\", \"gutter\", \"story\", \"hills\", \"enter\", \"nasa\", \"durian\", \"real\", \"gut\", \"biden\", \"time\", \"asleep\", \"vaccination\", \"shop\", \"gas\", \"first\", \"malaysian\", \"skit\", \"take\", \"narrative\", \"cover\", \"vietnam\", \"shukla\", \"helicopter\", \"admit\", \"siddhartha\", \"yoga\", \"hui\", \"balika\", \"reappear\", \"late\", \"flex\", \"last\", \"hang\", \"pakistani\", \"threaten\", \"sidharth_shukla\", \"sidharth\", \"moment\", \"indian_media\", \"man\", \"taliban\", \"modi\", \"show\", \"hindu\", \"pakistan\", \"pm\"], \"Freq\": [20.0, 10.0, 7.0, 6.0, 5.0, 5.0, 8.0, 4.0, 5.0, 4.0, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 4.9958985982764155, 3.996718878621132, 2.9975391589658487, 2.9975391589658487, 2.9975391589658487, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 5.995078317931697, 3.996718878621132, 1.998359439310566, 1.998359439310566, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 5.0007657757163075, 5.0007657757163075, 4.000612620573046, 4.000612620573046, 3.0004594654297843, 3.0004594654297843, 3.0004594654297843, 2.000306310286523, 2.000306310286523, 2.000306310286523, 2.000306310286523, 2.000306310286523, 3.0004594654297843, 3.0004594654297843, 3.0004594654297843, 3.0004594654297843, 2.0003995302684916, 2.0003995302684916, 2.0003995302684916, 1.0001997651342458, 1.0001997651342458, 1.0001997651342458, 2.0003995302684916, 2.0003995302684916, 1.0001997651342458, 1.0001997651342458, 7.001398355939722, 4.000799060536983, 4.000799060536983, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 5.000998825671229, 8.001598121073966, 4.000799060536983, 2.00217182289636, 2.00217182289636, 2.00217182289636, 1.00108591144818, 1.00108591144818, 1.00108591144818, 1.00108591144818, 1.00108591144818, 2.00217182289636, 1.00108591144818, 4.00434364579272, 4.00434364579272, 3.00325773434454, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 6.997160630304906, 4.997971878789218, 0.9995943757578437, 1.9991887515156874, 0.9995943757578437, 0.9995943757578437, 0.9995943757578437, 0.9995943757578437, 0.9995943757578437, 0.9995943757578437, 5.9975662545470625, 3.998377503031375, 3.998377503031375, 3.998377503031375, 2.9987831272735312, 2.9987831272735312, 2.9987831272735312, 2.9987831272735312, 7.99675500606275, 16.993104387883342, 4.997971878789218, 3.998377503031375, 3.998377503031375, 4.997971878789218, 3.998377503031375], \"Total\": [20.0, 10.0, 7.0, 6.0, 5.0, 5.0, 8.0, 4.0, 5.0, 4.0, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 0.999179719655283, 4.9958985982764155, 3.996718878621132, 2.9975391589658487, 2.9975391589658487, 2.9975391589658487, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 1.998359439310566, 10.993050196720915, 5.996466409522237, 1.998359439310566, 3.9987589695790575, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 1.0001531551432614, 5.0007657757163075, 5.0007657757163075, 4.000612620573046, 4.000612620573046, 3.0004594654297843, 3.0004594654297843, 3.0004594654297843, 2.000306310286523, 2.000306310286523, 2.000306310286523, 2.000306310286523, 2.000306310286523, 4.00065923056403, 4.0015453768779645, 4.999648216945472, 4.999648216945472, 2.0003995302684916, 2.0003995302684916, 2.0003995302684916, 1.0001997651342458, 1.0001997651342458, 1.0001997651342458, 2.0003995302684916, 2.0003995302684916, 1.0001997651342458, 1.0001997651342458, 7.001398355939722, 4.000799060536983, 4.000799060536983, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 3.000599295402738, 6.001151980814491, 11.99956096800278, 20.993903448420326, 2.00217182289636, 2.00217182289636, 2.00217182289636, 1.00108591144818, 1.00108591144818, 1.00108591144818, 1.00108591144818, 1.00108591144818, 2.00217182289636, 1.00108591144818, 4.00434364579272, 4.00434364579272, 3.00325773434454, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 2.00217182289636, 3.0023249780396215, 2.00217182289636, 2.00217182289636, 6.997160630304906, 4.997971878789218, 0.9995943757578437, 1.9991887515156874, 0.9995943757578437, 0.9995943757578437, 0.9995943757578437, 0.9995943757578437, 0.9995943757578437, 0.9995943757578437, 5.9975662545470625, 3.998377503031375, 3.998377503031375, 3.998377503031375, 2.9987831272735312, 2.9987831272735312, 2.9987831272735312, 2.9987831272735312, 8.99784091751093, 20.993903448420326, 5.99812503393248, 4.99857726816562, 4.99857726816562, 10.993050196720915, 4.999463414479555], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\"], \"logprob\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -5.0304, -5.0304, -5.0304, -5.0304, -5.0304, -5.0304, -5.0304, -5.0304, -5.0304, -5.0304, -3.421, -3.6441, -3.9318, -3.9318, -3.9318, -4.3373, -4.3373, -4.3373, -4.3373, -4.3373, -4.3373, -4.3373, -4.3373, -3.2387, -3.6441, -4.3373, -4.3373, -4.9558, -4.9558, -4.9558, -4.9558, -4.9558, -4.9558, -4.9558, -4.9558, -4.9558, -4.9558, -3.3464, -3.3464, -3.5695, -3.5695, -3.8572, -3.8572, -3.8572, -4.2627, -4.2627, -4.2627, -4.2627, -4.2627, -3.8572, -3.8572, -3.8572, -3.8572, -4.6299, -4.6299, -4.6299, -5.323, -5.323, -5.323, -4.6299, -4.6299, -5.323, -5.323, -3.3771, -3.9367, -3.9367, -4.2244, -4.2244, -4.2244, -4.2244, -4.2244, -4.2244, -4.2244, -4.2244, -4.2244, -3.7136, -3.2436, -3.9367, -4.2697, -4.2697, -4.2697, -4.9628, -4.9628, -4.9628, -4.9628, -4.9628, -4.2697, -4.9628, -3.5766, -3.5766, -3.8642, -4.2697, -4.2697, -4.2697, -4.2697, -4.2697, -4.2697, -4.2697, -4.2697, -4.2697, -4.2697, -3.4834, -3.8199, -5.4293, -4.7362, -5.4293, -5.4293, -5.4293, -5.4293, -5.4293, -5.4293, -3.6376, -4.0431, -4.0431, -4.0431, -4.3307, -4.3307, -4.3307, -4.3307, -3.3499, -2.5961, -3.8199, -4.0431, -4.0431, -3.8199, -4.0431], \"loglift\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.74, 1.1337, 1.3343, 1.74, 1.0464, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.8137, 1.526, 1.5257, 1.3031, 1.3031, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.4464, 1.2641, 1.0412, -0.2113, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.8057, 1.4006, 1.8057, 1.8057, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.3407, 1.2228, 1.1293, 1.1583, 1.1174, 1.1174, 0.5525, 1.1173]}, \"token.table\": {\"Topic\": [2, 5, 2, 1, 5, 1, 3, 5, 1, 3, 1, 2, 2, 2, 4, 1, 5, 4, 4, 2, 3, 1, 3, 3, 2, 2, 2, 4, 3, 2, 1, 4, 3, 4, 2, 4, 2, 3, 3, 4, 5, 2, 2, 4, 1, 2, 5, 3, 4, 4, 5, 1, 5, 2, 4, 4, 3, 5, 5, 1, 1, 5, 1, 3, 3, 3, 5, 5, 2, 1, 4, 4, 5, 2, 2, 2, 5, 5, 1, 2, 4, 4, 2, 1, 1, 2, 1, 5, 5, 2, 1, 4, 5, 1, 3, 3, 2, 3, 4, 5, 2, 1, 1, 2, 3, 3, 1, 3, 4, 3, 5, 5, 5, 5, 5, 4, 2, 3, 1, 4, 1, 3, 2, 5, 4, 3, 5, 3, 3, 5, 4, 1, 2, 3, 1, 3, 4, 4, 3, 1, 5, 1], \"Freq\": [0.600042216936784, 0.40002814462452263, 0.9998468683096445, 1.0008209537569477, 1.000405788839947, 0.08333638227819605, 0.6666910582255684, 0.25000914683458814, 1.0008209537569477, 0.9998002747638928, 1.0008209537569477, 0.9998468683096445, 0.9998468683096445, 0.9998468683096445, 0.9989152664763715, 1.000820953756948, 1.000405788839947, 0.9989152664763715, 0.9989152664763715, 0.9998468683096445, 0.9998002747638925, 1.000820953756948, 0.9998002747638928, 0.9998002747638928, 0.9998468683096444, 0.9998468683096445, 0.9998468683096445, 0.9989152664763715, 0.9998002747638928, 0.9998468683096445, 1.0008209537569477, 0.9989152664763715, 0.9998002747638928, 0.9989152664763715, 0.9998468683096445, 0.9989152664763715, 0.7498764146370566, 0.24995880487901884, 0.9998002747638925, 0.9989152664763715, 1.000405788839947, 0.9998468683096444, 0.9998468683096445, 0.9989152664763715, 0.6670595191941876, 0.1667648797985469, 0.1667648797985469, 0.9998002747638928, 0.9989152664763715, 0.9989152664763715, 1.000405788839947, 1.0008209537569477, 1.0004057888399471, 0.7497103537385404, 0.24990345124618016, 0.9989152664763715, 0.20005692547131124, 0.800227701885245, 1.000405788839947, 1.000820953756948, 1.0008209537569477, 1.000405788839947, 1.0008209537569477, 0.9998002747638925, 0.9998002747638928, 0.9998002747638928, 1.000405788839947, 1.000405788839947, 0.9998468683096445, 1.0008209537569477, 0.9989152664763715, 0.11113777284658082, 0.8891021827726465, 0.9998468683096445, 0.9998468683096445, 0.16671876533797125, 0.8335938266898563, 1.000405788839947, 1.0008209537569477, 0.3330752024895564, 0.6661504049791128, 0.9989152664763715, 0.9998468683096445, 1.0008209537569477, 1.0008209537569477, 0.9998468683096445, 0.5457993816665845, 0.45483281805548703, 1.000405788839947, 0.9998468683096445, 1.0008209537569477, 0.20002146572445717, 0.8000858628978287, 0.5001551769474459, 0.5001551769474459, 0.9998002747638925, 0.9998468683096445, 0.9998002747638925, 0.9989152664763715, 1.000405788839947, 0.9998468683096445, 1.0008209537569477, 1.0008209537569477, 0.16663467334221346, 0.8331733667110673, 0.9998002747638925, 1.0008209537569477, 0.9998002747638928, 0.9989152664763715, 0.20005692547131124, 0.800227701885245, 1.000405788839947, 1.000405788839947, 1.000405788839947, 1.000405788839947, 0.9989152664763715, 0.9998468683096445, 0.9998002747638925, 1.0008209537569477, 0.9989152664763715, 1.0008209537569477, 0.9998002747638928, 0.600042216936784, 0.40002814462452263, 0.9989152664763715, 0.19053150405438193, 0.8097588922311232, 0.9998002747638925, 0.9998002747638925, 1.000405788839947, 0.9989152664763715, 1.0008209537569477, 0.9998468683096445, 0.9998002747638928, 1.0008209537569477, 0.9998002747638928, 0.9989152664763715, 0.9989152664763715, 0.9998002747638925, 1.0008209537569477, 1.000405788839947, 1.0008209537569477], \"Term\": [\"aaj\", \"aaj\", \"aajtak\", \"adalat\", \"admit\", \"afghanistan\", \"afghanistan\", \"afghanistan\", \"afghans\", \"alcohol\", \"ambedkar\", \"anil\", \"apologise\", \"apologize\", \"asleep\", \"asset\", \"balika\", \"biden\", \"bird\", \"bjp\", \"book\", \"border\", \"christian\", \"classroom\", \"congress\", \"cost\", \"court\", \"cover\", \"cow\", \"cristiano\", \"crore\", \"dalit\", \"damage\", \"durian\", \"election\", \"enter\", \"family\", \"family\", \"farmer\", \"first\", \"flex\", \"gandhi\", \"gandhi_family\", \"gas\", \"government\", \"government\", \"government\", \"gujarat\", \"gut\", \"gutter\", \"hang\", \"happen\", \"helicopter\", \"high\", \"high\", \"hills\", \"hindu\", \"hindu\", \"hui\", \"hyderabad\", \"hyderabad_traffic\", \"indian_media\", \"jan\", \"journalist\", \"kabul\", \"kill\", \"last\", \"late\", \"lawyer\", \"lok\", \"malaysian\", \"man\", \"man\", \"manchester\", \"mla\", \"modi\", \"modi\", \"moment\", \"murdabad\", \"narrative\", \"narrative\", \"nasa\", \"newsletter\", \"oct\", \"official\", \"order\", \"pakistan\", \"pakistan\", \"pakistani\", \"path\", \"photograph\", \"pm\", \"pm\", \"police\", \"police\", \"protest\", \"rahul\", \"ratan\", \"real\", \"reappear\", \"reliance\", \"round\", \"sabha\", \"school\", \"school\", \"school_books\", \"shape\", \"shared_visuals\", \"shop\", \"show\", \"show\", \"shukla\", \"siddhartha\", \"sidharth\", \"sidharth_shukla\", \"skit\", \"speak\", \"srinagar\", \"sticker\", \"story\", \"suzuki\", \"syria\", \"tak\", \"tak\", \"take\", \"taliban\", \"taliban\", \"tata\", \"tax\", \"threaten\", \"time\", \"traffic\", \"tweet\", \"us\", \"uzjin\", \"vaccinate\", \"vaccination\", \"vietnam\", \"visual\", \"wound\", \"yoga\", \"zindabad\"]}, \"R\": 10, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [1, 2, 3, 4, 5]};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el96947434267844708335610\", ldavis_el96947434267844708335610_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el96947434267844708335610\", ldavis_el96947434267844708335610_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el96947434267844708335610\", ldavis_el96947434267844708335610_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 429,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabulary = list(vocab)\n",
    "doc_topic_dists = [mgp.score(doc) for doc in data_cleaned]\n",
    "doc_lengths = [len(doc) for doc in data_cleaned]\n",
    "term_counts_map = {}\n",
    "for doc in data_cleaned:\n",
    "    for term in doc:\n",
    "        term_counts_map[term] = term_counts_map.get(term, 0) + 1\n",
    "term_counts = [term_counts_map[term] for term in vocabulary]\n",
    "\n",
    "matrix = []\n",
    "for cluster in mgp.cluster_word_distribution:\n",
    "    total = sum([occurence for word, occurence in cluster.items()])\n",
    "    row = [cluster.get(term, 0) / total for term in vocabulary]\n",
    "    matrix.append(row)\n",
    "\n",
    "vis_data = pyLDAvis.prepare(topic_term_dists=matrix, doc_topic_dists=doc_topic_dists, doc_lengths=doc_lengths, \n",
    "                            vocab=vocabulary, R=10, term_frequency=term_counts, sort_topics=False)\n",
    "pyLDAvis.enable_notebook()\n",
    "pyLDAvis.display(vis_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 430,
   "metadata": {},
   "outputs": [],
   "source": [
    "pyLDAvis.save_json(vis_data, \"2021_wk35.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 431,
   "metadata": {},
   "outputs": [],
   "source": [
    "pyLDAvis.save_html(vis_data, \"2021_wk35.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add headline links and article count to json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"2021_wk35.json\", \"r\") as f:\n",
    "    data = json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"per_cluster_headlines\"] = headlines_dict\n",
    "data[\"numpy_seed\"] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 434,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"number_of_articles\"] = len(clean_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 435,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save file that will be used in the themes dashboard\n",
    "with open(\"2021_wk35.json\", \"w\") as f:\n",
    "    json.dump(data, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recreate viz from json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 787,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepared_data_from_dict(vis_data):\n",
    "    topic_coordinates = pd.DataFrame.from_dict(vis_data['mdsDat'])\n",
    "    topic_info = pd.DataFrame.from_dict(vis_data['tinfo'])\n",
    "    token_table = pd.DataFrame.from_dict(vis_data['token.table'])\n",
    "    R = vis_data['R']\n",
    "    lambda_step = vis_data['lambda.step']\n",
    "    plot_opts = vis_data['plot.opts']\n",
    "    client_topic_order = vis_data['topic.order']\n",
    "\n",
    "    return PreparedData(topic_coordinates, topic_info,\n",
    "                        token_table, R, lambda_step, plot_opts, client_topic_order)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 802,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el731249219690088420452271\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el731249219690088420452271_data = {\"mdsDat\": {\"x\": [0.13790267238838402, 0.18479678238890998, -0.3486443994487772, 0.19425582530913593, -0.16831088063765248], \"y\": [-0.18656913724524682, 0.3795099505594992, 0.08060412453895813, -0.17194697390904673, -0.10159796394416387], \"topics\": [1, 2, 3, 4, 5], \"cluster\": [1, 1, 1, 1, 1], \"Freq\": [20.688455217992423, 18.390788067188375, 17.370425351732983, 25.670640995146883, 17.87969036793934]}, \"tinfo\": {\"Term\": [\"pranab\", \"mukherjee\", \"tweet\", \"burn\", \"rana\", \"communal\", \"death\", \"pranab_mukherjee\", \"angle\", \"journalist\", \"pm\", \"afghanistan\", \"kangana\", \"religion\", \"donkey\", \"married\", \"bjp\", \"jinpe\", \"collector\", \"homage\", \"kapil\", \"sister\", \"mishra\", \"kapil_mishra\", \"hussein\", \"describe\", \"praise\", \"see\", \"body\", \"man\", \"year\", \"district\", \"modi\", \"rahul\", \"indore\", \"muslim_man\", \"rhea\", \"amit\", \"us\", \"rhea_chakraborty\", \"chakraborty\", \"june\", \"terrorist\", \"positive\", \"slip\", \"poll\", \"shah\", \"hit\", \"presidential\", \"believe\", \"jee\", \"report\", \"karnataka\", \"go\", \"fall\", \"das\", \"census\", \"tweet\", \"thieve\", \"mukherjee\", \"explosion\", \"manipulate\", \"robbery\", \"accuse\", \"ankhi\", \"pranab\", \"rana\", \"pranab_mukherjee\", \"journalist\", \"rana_ayub\", \"tweet_pranab\", \"rana_ayyub\", \"flight\", \"death\", \"first\", \"ayyub\", \"switzerland\", \"article\", \"publish\", \"attack\", \"virus\", \"chennai\", \"love\", \"maa\", \"amar\", \"hyderabad\", \"burn\", \"communal\", \"angle\", \"flag\", \"kali\", \"idol\", \"bengal\", \"west\", \"incident\", \"murder\", \"west_bengal\", \"muslim\", \"police\", \"pla\", \"time\", \"china\", \"occupy\", \"mughal\", \"islam\", \"take\", \"frontier\", \"standoff\", \"scatter\", \"sweden\", \"soldier\", \"galwan\", \"chinese\", \"riot\", \"valley\", \"injure\", \"group\", \"kill\", \"garden\", \"army\", \"memorial\", \"indian\", \"times\", \"late\", \"date\", \"unrelated\", \"muslims\", \"show\"], \"Freq\": [8.0, 8.0, 7.0, 8.0, 6.0, 7.0, 7.0, 5.0, 6.0, 4.0, 2.999826006608902, 0.9999420022029673, 0.9999420022029673, 0.9999420022029673, 0.9999420022029673, 1.9998840044059345, 0.9999420022029673, 0.9999420022029673, 0.9999420022029673, 0.9999420022029673, 3.999768008811869, 3.999768008811869, 2.999826006608902, 2.999826006608902, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 4.9997100110148365, 2.999826006608902, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 2.9999973034601037, 2.9999973034601037, 2.9999973034601037, 2.9999973034601037, 2.9999973034601037, 0.9999991011533679, 1.9999982023067358, 0.9999991011533679, 0.9999991011533679, 0.9999991011533679, 3.9999964046134715, 1.9999982023067358, 1.9999982023067358, 1.9999982023067358, 1.9999982023067358, 1.9999982023067358, 1.9999982023067358, 2.9999973034601037, 1.9999982023067358, 1.0000766948828623, 2.0001533897657247, 7.000536864180036, 1.0000766948828623, 8.000613559062899, 1.0000766948828623, 1.0000766948828623, 1.0000766948828623, 1.0000766948828623, 1.0000766948828623, 8.000613559062899, 6.000460169297175, 5.0003834744143125, 4.000306779531449, 3.0002300846485874, 3.0002300846485874, 3.0002300846485874, 3.0002300846485874, 6.000460169297175, 3.0002300846485874, 3.0002300846485874, 1.0000055671243786, 1.0000055671243786, 1.0000055671243786, 4.000022268497514, 1.0000055671243786, 2.000011134248757, 3.0000167013731356, 1.0000055671243786, 1.0000055671243786, 3.0000167013731356, 8.000044536995029, 7.00003896987065, 6.000033402746271, 5.000027835621894, 5.000027835621894, 5.000027835621894, 4.000022268497514, 4.000022268497514, 4.000022268497514, 4.000022268497514, 4.000022268497514, 6.000033402746271, 5.000027835621894, 0.9999855398640362, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 0.9999855398640362, 0.9999855398640362, 0.9999855398640362, 0.9999855398640362, 0.9999855398640362, 0.9999855398640362, 3.9999421594561446, 3.9999421594561446, 3.9999421594561446, 3.9999421594561446, 2.9999566195921084, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 3.9999421594561446, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 2.9999566195921084, 2.9999566195921084, 2.9999566195921084], \"Total\": [8.0, 8.0, 7.0, 8.0, 6.0, 7.0, 7.0, 5.0, 6.0, 4.0, 2.999826006608902, 0.9999420022029673, 0.9999420022029673, 0.9999420022029673, 0.9999420022029673, 1.9998840044059345, 0.9999420022029673, 0.9999420022029673, 0.9999420022029673, 0.9999420022029673, 3.999768008811869, 3.999768008811869, 2.999826006608902, 2.999826006608902, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 6.999721145263594, 3.999831573733281, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 1.9998840044059345, 2.9998895715303133, 2.9999973034601037, 2.9999973034601037, 2.9999973034601037, 2.9999973034601037, 2.9999973034601037, 0.9999991011533679, 1.9999982023067358, 0.9999991011533679, 0.9999991011533679, 0.9999991011533679, 3.9999964046134715, 1.9999982023067358, 1.9999982023067358, 1.9999982023067358, 1.9999982023067358, 1.9999982023067358, 1.9999982023067358, 5.0000795654673444, 3.9998822067126705, 1.0000766948828623, 2.0001533897657247, 7.000536864180036, 1.0000766948828623, 8.000613559062899, 1.0000766948828623, 1.0000766948828623, 1.0000766948828623, 1.0000766948828623, 1.0000766948828623, 8.000613559062899, 6.000460169297175, 5.0003834744143125, 4.000306779531449, 3.0002300846485874, 3.0002300846485874, 3.0002300846485874, 3.0002300846485874, 7.000402171500142, 3.0002300846485874, 3.0002300846485874, 1.0000055671243786, 1.0000055671243786, 1.0000055671243786, 4.000022268497514, 1.0000055671243786, 2.000011134248757, 3.0000167013731356, 1.0000055671243786, 1.0000055671243786, 3.0000167013731356, 8.000044536995029, 7.00003896987065, 6.000033402746271, 5.000027835621894, 5.000027835621894, 5.000027835621894, 4.000022268497514, 4.000022268497514, 4.000022268497514, 4.000022268497514, 4.000022268497514, 7.999917407152205, 6.000026936775262, 0.9999855398640362, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 0.9999855398640362, 0.9999855398640362, 0.9999855398640362, 0.9999855398640362, 0.9999855398640362, 0.9999855398640362, 3.9999421594561446, 3.9999421594561446, 3.9999421594561446, 3.9999421594561446, 2.9999566195921084, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 6.00009554922187, 1.9999710797280723, 1.9999710797280723, 1.9999710797280723, 4.999961287869855, 5.999973320965244, 5.9999744178313055], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\"], \"logprob\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -3.989, -5.0876, -5.0876, -5.0876, -5.0876, -4.3944, -5.0876, -5.0876, -5.0876, -5.0876, -3.7013, -3.7013, -3.989, -3.989, -4.3944, -4.3944, -4.3944, -4.3944, -4.3944, -3.4782, -3.989, -4.3944, -4.3944, -4.3944, -4.3944, -4.3944, -3.8712, -3.8712, -3.8712, -3.8712, -3.8712, -4.9698, -4.2767, -4.9698, -4.9698, -4.9698, -3.5835, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -3.8712, -4.2767, -4.9127, -4.2195, -2.9667, -4.9127, -2.8332, -4.9127, -4.9127, -4.9127, -4.9127, -4.9127, -2.8332, -3.1209, -3.3032, -3.5264, -3.814, -3.814, -3.814, -3.814, -3.1209, -3.814, -3.814, -5.3033, -5.3033, -5.3033, -3.917, -5.3033, -4.6102, -4.2047, -5.3033, -5.3033, -4.2047, -3.2239, -3.3574, -3.5115, -3.6939, -3.6939, -3.6939, -3.917, -3.917, -3.917, -3.917, -3.917, -3.5115, -3.6939, -4.9416, -4.2485, -4.2485, -4.2485, -4.9416, -4.9416, -4.9416, -4.9416, -4.9416, -4.9416, -3.5553, -3.5553, -3.5553, -3.5553, -3.843, -4.2485, -4.2485, -4.2485, -4.2485, -4.2485, -4.2485, -4.2485, -3.5553, -4.2485, -4.2485, -4.2485, -3.843, -3.843, -3.843], \"loglift\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.5756, 1.2391, 1.2879, 1.5756, 1.5756, 1.5756, 1.5756, 1.1701, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.6933, 1.1825, 1.0002, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.7504, 1.5963, 1.7504, 1.7504, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.3598, 1.0722, 1.1775, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.7215, 1.316, 1.7215, 1.7215, 1.7215, 1.2107, 1.0283, 1.0283]}, \"token.table\": {\"Topic\": [3, 1, 4, 2, 4, 3, 5, 4, 4, 3, 2, 4, 1, 1, 4, 3, 2, 4, 5, 5, 1, 4, 3, 5, 1, 3, 1, 1, 1, 3, 1, 2, 3, 4, 3, 5, 5, 5, 2, 3, 4, 5, 2, 1, 1, 4, 4, 4, 3, 5, 1, 5, 5, 2, 1, 3, 2, 4, 1, 1, 1, 2, 5, 5, 4, 4, 1, 4, 3, 1, 5, 1, 1, 5, 3, 4, 1, 4, 1, 4, 4, 5, 5, 5, 1, 2, 4, 2, 2, 1, 3, 3, 2, 4, 1, 3, 3, 3, 1, 2, 2, 2, 5, 3, 5, 1, 2, 1, 2, 3, 5, 1, 2, 5, 5, 5, 4, 5, 2, 3, 5, 5, 3, 3, 2, 4, 5, 2, 5, 4, 4, 4, 1, 4], \"Freq\": [0.9999233109987916, 1.0000580011609723, 0.999994432906614, 1.00000089884744, 0.9999944329066142, 0.9999233109987916, 1.0000144603450625, 0.999994432906614, 0.999994432906614, 0.9999233109987915, 1.00000089884744, 0.999994432906614, 1.0000580011609723, 1.0000580011609723, 0.999994432906614, 0.9999233109987916, 1.00000089884744, 0.999994432906614, 1.0000144603450625, 1.0000144603450625, 1.0000580011609723, 0.999994432906614, 0.9999233109987916, 1.0000144603450625, 0.14284893574703098, 0.8570936144821859, 1.0000580011609723, 1.0000580011609723, 1.0000580011609723, 0.9999233109987916, 0.5000147245945308, 0.5000147245945308, 0.9999233109987915, 0.9999944329066139, 0.9999233109987915, 1.0000144603450625, 1.0000144603450625, 1.0000144603450625, 0.5999904522958521, 0.19999681743195072, 0.19999681743195072, 1.0000144603450625, 1.00000089884744, 1.0000580011609723, 1.0000580011609723, 0.9999944329066142, 0.9999944329066139, 0.999994432906614, 0.33332802512776194, 0.6666560502555239, 1.0000580011609723, 1.0000144603450625, 1.0000144603450625, 1.00000089884744, 1.0000580011609723, 0.9999233109987916, 1.00000089884744, 0.9999944329066139, 1.0000580011609723, 1.0000580011609723, 1.0000580011609723, 1.00000089884744, 1.0000144603450625, 1.0000144603450625, 0.9999944329066142, 0.999994432906614, 0.714314169984226, 0.28572566799369037, 0.9999233109987916, 1.0000580011609723, 1.0000144603450625, 1.0000580011609723, 1.0000580011609723, 1.0000144603450625, 0.9999233109987916, 0.999994432906614, 0.25000258105314066, 0.750007743159422, 0.6666912072299227, 0.33334560361496135, 0.5000022232627821, 0.5000022232627821, 1.0000144603450625, 1.0000144603450625, 1.0000580011609723, 0.1666659184262686, 0.8333295921313429, 1.00000089884744, 1.00000089884744, 1.0000580011609723, 0.9999233109987916, 0.9999233109987915, 1.00000089884744, 0.999994432906614, 1.0000580011609723, 0.9999233109987915, 0.9999233109987915, 0.9999233109987915, 1.0000580011609723, 1.00000089884744, 1.00000089884744, 1.00000089884744, 1.0000144603450625, 0.9999233109987916, 1.0000144603450625, 1.0000580011609723, 1.00000089884744, 0.1666673772854936, 0.1666673772854936, 0.1666673772854936, 0.5000021318564808, 1.0000580011609723, 1.00000089884744, 1.0000144603450625, 1.0000144603450625, 1.0000144603450625, 0.999994432906614, 1.0000144603450625, 1.00000089884744, 0.9999233109987916, 1.0000144603450625, 1.0000144603450625, 0.9999233109987916, 0.9999233109987915, 0.20000154849719493, 0.20000154849719493, 0.6000046454915848, 1.00000089884744, 1.0000144603450625, 0.999994432906614, 0.999994432906614, 0.999994432906614, 0.7500315812547881, 0.2500105270849294], \"Term\": [\"accuse\", \"afghanistan\", \"amar\", \"amit\", \"angle\", \"ankhi\", \"army\", \"article\", \"attack\", \"ayyub\", \"believe\", \"bengal\", \"bjp\", \"body\", \"burn\", \"census\", \"chakraborty\", \"chennai\", \"china\", \"chinese\", \"collector\", \"communal\", \"das\", \"date\", \"death\", \"death\", \"describe\", \"district\", \"donkey\", \"explosion\", \"fall\", \"fall\", \"first\", \"flag\", \"flight\", \"frontier\", \"galwan\", \"garden\", \"go\", \"go\", \"go\", \"group\", \"hit\", \"homage\", \"hussein\", \"hyderabad\", \"idol\", \"incident\", \"indian\", \"indian\", \"indore\", \"injure\", \"islam\", \"jee\", \"jinpe\", \"journalist\", \"june\", \"kali\", \"kangana\", \"kapil\", \"kapil_mishra\", \"karnataka\", \"kill\", \"late\", \"love\", \"maa\", \"man\", \"man\", \"manipulate\", \"married\", \"memorial\", \"mishra\", \"modi\", \"mughal\", \"mukherjee\", \"murder\", \"muslim\", \"muslim\", \"muslim_man\", \"muslim_man\", \"muslims\", \"muslims\", \"occupy\", \"pla\", \"pm\", \"police\", \"police\", \"poll\", \"positive\", \"praise\", \"pranab\", \"pranab_mukherjee\", \"presidential\", \"publish\", \"rahul\", \"rana\", \"rana_ayub\", \"rana_ayyub\", \"religion\", \"report\", \"rhea\", \"rhea_chakraborty\", \"riot\", \"robbery\", \"scatter\", \"see\", \"shah\", \"show\", \"show\", \"show\", \"show\", \"sister\", \"slip\", \"soldier\", \"standoff\", \"sweden\", \"switzerland\", \"take\", \"terrorist\", \"thieve\", \"time\", \"times\", \"tweet\", \"tweet_pranab\", \"unrelated\", \"unrelated\", \"unrelated\", \"us\", \"valley\", \"virus\", \"west\", \"west_bengal\", \"year\", \"year\"]}, \"R\": 10, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [1, 2, 3, 4, 5]};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el731249219690088420452271\", ldavis_el731249219690088420452271_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el731249219690088420452271\", ldavis_el731249219690088420452271_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el731249219690088420452271\", ldavis_el731249219690088420452271_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 802,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('wk37.json', 'r') as json_file:\n",
    "    dict_data = json.load(json_file)\n",
    "    viz_data = prepared_data_from_dict(dict_data)\n",
    "pyLDAvis.display(viz_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 803,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el731249032592649826915758\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el731249032592649826915758_data = {\"mdsDat\": {\"x\": [-0.08600446833785515, -0.3342305153232342, -0.018051022816722707, 0.3187679775221782, 0.11951802895563356], \"y\": [0.3888718272971789, -0.15927860831127238, -0.19592010501292026, -0.09687867320138831, 0.06320555922840168], \"topics\": [1, 2, 3, 4, 5], \"cluster\": [1, 1, 1, 1, 1], \"Freq\": [15.106185964244185, 14.326775599811867, 21.443013547464346, 24.812106293653013, 24.31191859482659]}, \"tinfo\": {\"Term\": [\"modi\", \"bengal\", \"pm_modi\", \"pm\", \"west_bengal\", \"west\", \"protest\", \"covid\", \"facebook\", \"das\", \"aankhi\", \"bababhai\", \"zealand\", \"kangana\", \"terrorist\", \"policy\", \"cake\", \"lashkar\", \"collapse\", \"ambassador\", \"das\", \"facebook\", \"khan\", \"aamir_khan\", \"aamir\", \"cut\", \"ankhi\", \"ankhi_das\", \"gurugram\", \"indian\", \"force\", \"colonial\", \"sydney\", \"chakraborty\", \"internet\", \"links\", \"billboard\", \"retirement\", \"sewage\", \"bomb\", \"delhi\", \"truck\", \"mandir\", \"varanasi\", \"australia\", \"fuji\", \"ram\", \"rhea\", \"bhatt\", \"israel\", \"uttar\", \"fall\", \"bounce\", \"rs\", \"visual\", \"road\", \"morphed\", \"find\", \"pm_modi\", \"pandemic\", \"drink\", \"predict\", \"pr\", \"lk\", \"housewife\", \"ibrahim\", \"duck\", \"municipality\", \"message\", \"mother\", \"mercury\", \"pay\", \"pm\", \"red\", \"modi\", \"minister\", \"gandhi\", \"covid\", \"modi_duck\", \"link\", \"corona\", \"dose\", \"comed\", \"celebrate\", \"party\", \"testing\", \"vaccine\", \"want\", \"galwan_valley\", \"vp\", \"exam\", \"student\", \"japan\", \"tsunami\", \"china\", \"screenshot\", \"comedk\", \"galwan\", \"karnataka\", \"death\", \"valley\", \"positive\", \"pakistan\", \"flood\", \"ganesh\", \"covid\", \"man\", \"project\", \"teacher\", \"wrongly\", \"slum\", \"west_bengal\", \"spread\", \"morph\", \"faith\", \"slogans\", \"tweet\", \"bengal\", \"west\", \"protest\", \"bangladesh\", \"protest_bangladesh\", \"gdp\", \"mishra\", \"kapil\", \"imf\", \"growth\", \"kapil_mishra\", \"indian\", \"bjp\", \"show\"], \"Freq\": [17.0, 10.0, 10.0, 12.0, 9.0, 9.0, 8.0, 14.0, 6.0, 6.0, 2.0013290958998664, 1.0006645479499332, 3.0019936438497994, 2.0013290958998664, 2.0013290958998664, 1.0006645479499332, 4.002658191799733, 2.0013290958998664, 4.002658191799733, 2.0013290958998664, 6.003987287699599, 6.003987287699599, 5.003322739749666, 5.003322739749666, 5.003322739749666, 4.002658191799733, 3.0019936438497994, 3.0019936438497994, 3.0019936438497994, 4.002658191799733, 0.9999897062955934, 0.9999897062955934, 1.9999794125911867, 1.9999794125911867, 0.9999897062955934, 0.9999897062955934, 0.9999897062955934, 0.9999897062955934, 0.9999897062955934, 0.9999897062955934, 3.9999588251823734, 2.99996911888678, 2.99996911888678, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 3.0000987564165364, 3.0000987564165364, 10.00032918805512, 3.0000987564165364, 3.0000987564165364, 1.0000329188055121, 2.0000658376110243, 1.0000329188055121, 1.0000329188055121, 2.0000658376110243, 4.000131675222049, 2.0000658376110243, 2.0000658376110243, 2.0000658376110243, 2.0000658376110243, 2.0000658376110243, 11.000362106860631, 2.0000658376110243, 15.000493782082682, 2.0000658376110243, 2.0000658376110243, 9.000296269249608, 2.0000658376110243, 3.0000987564165364, 7.001245496813718, 1.0001779281162455, 1.0001779281162455, 1.0001779281162455, 1.0001779281162455, 1.0001779281162455, 2.000355856232491, 1.0001779281162455, 3.0005337843487365, 1.0001779281162455, 8.001423424929964, 6.001067568697473, 4.000711712464982, 4.000711712464982, 4.000711712464982, 4.000711712464982, 3.0005337843487365, 3.0005337843487365, 3.0005337843487365, 3.0005337843487365, 3.0005337843487365, 3.0005337843487365, 5.000889640581227, 3.0005337843487365, 3.0005337843487365, 5.000889640581227, 3.0005337843487365, 0.9993832149652037, 0.9993832149652037, 0.9993832149652037, 0.9993832149652037, 9.993832149652038, 0.9993832149652037, 0.9993832149652037, 0.9993832149652037, 0.9993832149652037, 1.9987664299304073, 10.99321536461724, 9.993832149652038, 8.994448934686835, 7.995065719721629, 3.9975328598608146, 3.9975328598608146, 3.9975328598608146, 2.998149644895611, 2.998149644895611, 2.998149644895611, 2.998149644895611, 4.996916074826019, 3.9975328598608146, 3.9975328598608146], \"Total\": [17.0, 10.0, 10.0, 12.0, 9.0, 9.0, 8.0, 14.0, 6.0, 6.0, 2.0013290958998664, 1.0006645479499332, 3.0019936438497994, 2.0013290958998664, 2.0013290958998664, 1.0006645479499332, 4.002658191799733, 2.0013290958998664, 4.002658191799733, 2.0013290958998664, 6.003987287699599, 6.003987287699599, 5.003322739749666, 5.003322739749666, 5.003322739749666, 4.002658191799733, 3.0019936438497994, 3.0019936438497994, 3.0019936438497994, 8.999574266625752, 0.9999897062955934, 0.9999897062955934, 1.9999794125911867, 1.9999794125911867, 0.9999897062955934, 0.9999897062955934, 0.9999897062955934, 0.9999897062955934, 0.9999897062955934, 0.9999897062955934, 3.9999588251823734, 2.99996911888678, 2.99996911888678, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 1.9999794125911867, 5.998811680132619, 1.9999794125911867, 4.000045250202211, 1.9999794125911867, 4.0000271755063235, 3.0000987564165364, 3.0000987564165364, 10.00032918805512, 3.0000987564165364, 3.0000987564165364, 1.0000329188055121, 2.0000658376110243, 1.0000329188055121, 1.0000329188055121, 2.0000658376110243, 4.000131675222049, 2.0000658376110243, 2.0000658376110243, 2.0000658376110243, 2.0000658376110243, 2.0000658376110243, 12.000351813156225, 2.0000658376110243, 17.000849638315174, 2.0000658376110243, 2.0000658376110243, 14.001185909830834, 2.0000658376110243, 6.9992210425794354, 7.001245496813718, 1.0001779281162455, 1.0001779281162455, 1.0001779281162455, 1.0001779281162455, 1.0001779281162455, 2.000355856232491, 1.0001779281162455, 3.0005337843487365, 1.0001779281162455, 8.001423424929964, 6.001067568697473, 4.000711712464982, 4.000711712464982, 4.000711712464982, 4.000711712464982, 3.0005337843487365, 3.0005337843487365, 3.0005337843487365, 3.0005337843487365, 3.0005337843487365, 3.0005337843487365, 6.000272855546431, 3.0005337843487365, 3.0005337843487365, 14.001185909830834, 5.9999647622290775, 0.9993832149652037, 0.9993832149652037, 0.9993832149652037, 0.9993832149652037, 9.993832149652038, 0.9993832149652037, 0.9993832149652037, 0.9993832149652037, 0.9993832149652037, 1.9987664299304073, 10.99321536461724, 9.993832149652038, 8.994448934686835, 7.995065719721629, 3.9975328598608146, 3.9975328598608146, 3.9975328598608146, 2.998149644895611, 2.998149644895611, 2.998149644895611, 2.998149644895611, 8.999574266625752, 5.9975986974718385, 8.998720898455078], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\"], \"logprob\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -4.3631, -5.0562, -3.9576, -4.3631, -4.3631, -5.0562, -3.67, -4.3631, -3.67, -4.3631, -3.2645, -3.2645, -3.4468, -3.4468, -3.4468, -3.67, -3.9576, -3.9576, -3.9576, -3.67, -5.0039, -5.0039, -4.3108, -4.3108, -5.0039, -5.0039, -5.0039, -5.0039, -5.0039, -5.0039, -3.6177, -3.9053, -3.9053, -4.3108, -4.3108, -4.3108, -4.3108, -4.3108, -4.3108, -4.3108, -4.3108, -4.3108, -4.3108, -4.3108, -4.3108, -4.3108, -4.3086, -4.3086, -3.1046, -4.3086, -4.3086, -5.4072, -4.714, -5.4072, -5.4072, -4.714, -4.0209, -4.714, -4.714, -4.714, -4.714, -4.714, -3.0093, -4.714, -2.6991, -4.714, -4.714, -3.2099, -4.714, -4.3086, -3.607, -5.553, -5.553, -5.553, -5.553, -5.553, -4.8598, -5.553, -4.4543, -5.553, -3.4735, -3.7612, -4.1667, -4.1667, -4.1667, -4.1667, -4.4543, -4.4543, -4.4543, -4.4543, -4.4543, -4.4543, -3.9435, -4.4543, -4.4543, -3.9435, -4.4543, -5.5334, -5.5334, -5.5334, -5.5334, -3.2308, -5.5334, -5.5334, -5.5334, -5.5334, -4.8402, -3.1355, -3.2308, -3.3362, -3.4539, -4.1471, -4.1471, -4.1471, -4.4348, -4.4348, -4.4348, -4.4348, -3.924, -4.1471, -4.1471], \"loglift\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.8901, 1.0798, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 1.943, 0.8446, 1.943, 1.2499, 1.943, 1.2499, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.5398, 1.4528, 1.5398, 1.4146, 1.5398, 1.5398, 1.0979, 1.5398, 0.6926, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.3938, 1.2116, 1.3938, 1.3938, 0.3643, 0.7009, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 1.4142, 0.8258, 1.0085, 0.6028]}, \"token.table\": {\"Topic\": [1, 1, 1, 1, 1, 1, 2, 1, 5, 5, 2, 2, 3, 5, 2, 2, 1, 4, 2, 4, 1, 2, 4, 4, 4, 3, 4, 1, 1, 4, 2, 4, 3, 3, 4, 1, 5, 2, 3, 5, 3, 4, 2, 2, 4, 4, 3, 4, 5, 5, 1, 3, 3, 5, 1, 5, 2, 2, 4, 1, 5, 5, 4, 1, 1, 3, 4, 5, 2, 3, 1, 4, 5, 2, 3, 3, 3, 5, 3, 4, 3, 5, 3, 3, 3, 4, 5, 3, 4, 3, 2, 3, 3, 1, 4, 3, 3, 5, 5, 5, 2, 3, 2, 2, 1, 2, 5, 2, 3, 4, 2, 1, 2, 4, 5, 5, 5, 5, 4, 2, 5, 1, 4, 2, 4, 5, 2, 4, 4, 2, 2, 4, 4, 5, 5, 5, 1], \"Freq\": [0.9993358933807592, 0.9993358933807592, 0.9993358933807591, 0.9993358933807591, 0.9993358933807592, 0.9993358933807592, 1.000010293810368, 0.9993358933807591, 1.00061716569336, 1.00061716569336, 1.000010293810368, 1.000010293810368, 0.33346679244195815, 0.6669335848839163, 1.000010293810368, 1.000010293810368, 0.9993358933807591, 0.9998221035365371, 1.000010293810368, 0.9998221035365371, 0.9993358933807591, 1.000010293810368, 0.9998221035365371, 0.9998221035365371, 0.9998221035365372, 0.642802692426269, 0.35711260690348273, 0.9993358933807591, 0.9993358933807592, 0.9998221035365371, 1.000010293810368, 0.9998221035365371, 0.9999670822780999, 0.9999670822780999, 0.9998221035365371, 0.9993358933807592, 1.00061716569336, 0.3333993641813715, 0.3333993641813715, 0.3333993641813715, 0.9999670822780999, 0.9998221035365371, 1.000010293810368, 1.000010293810368, 0.9998221035365371, 0.9998221035365371, 0.9999670822780999, 0.9998221035365371, 1.00061716569336, 1.00061716569336, 0.9993358933807592, 0.9999670822780999, 0.9999670822780999, 1.00061716569336, 0.4444654693093318, 0.5555818366366647, 1.000010293810368, 1.000010293810368, 0.9998221035365371, 0.9993358933807591, 1.00061716569336, 1.00061716569336, 0.9998221035365371, 0.9993358933807592, 0.9993358933807591, 0.42861912514973305, 0.2857460834331554, 0.2857460834331554, 1.000010293810368, 0.9999670822780999, 0.1666676454993854, 0.5000029364981562, 0.3333352909987708, 1.000010293810368, 0.9999670822780999, 0.9999670822780999, 0.9999670822780999, 1.00061716569336, 0.882308844505876, 0.11764117926745013, 0.9999670822780999, 1.00061716569336, 0.9999670822780999, 0.9999670822780999, 0.9999670822780999, 0.833295438452967, 0.16665908769059343, 0.9999670822780999, 0.9998221035365371, 0.9999670822780999, 0.08333089025804061, 0.9166397928384467, 0.9999670822781, 0.9993358933807591, 0.9998221035365371, 0.9999670822780999, 0.9999670822780999, 1.00061716569336, 1.0006171656933598, 1.00061716569336, 1.000010293810368, 0.9999670822780999, 1.000010293810368, 1.000010293810368, 0.24999830154239389, 0.49999660308478777, 0.24999830154239389, 0.49999434378870977, 0.49999434378870977, 0.9998221035365371, 1.000010293810368, 0.11112690473283625, 0.11112690473283625, 0.33338071419850873, 0.444507618931345, 1.00061716569336, 1.00061716569336, 1.00061716569336, 0.9998221035365371, 1.000010293810368, 1.00061716569336, 0.9993358933807591, 0.9998221035365371, 1.000010293810368, 0.9998221035365371, 1.00061716569336, 1.000010293810368, 0.9998221035365371, 0.9998221035365371, 1.000010293810368, 1.000010293810368, 0.9998221035365371, 0.9998221035365371, 1.0006171656933598, 1.0006171656933598, 1.00061716569336, 0.9993358933807592], \"Term\": [\"aamir\", \"aamir_khan\", \"aankhi\", \"ambassador\", \"ankhi\", \"ankhi_das\", \"australia\", \"bababhai\", \"bangladesh\", \"bengal\", \"bhatt\", \"billboard\", \"bjp\", \"bjp\", \"bomb\", \"bounce\", \"cake\", \"celebrate\", \"chakraborty\", \"china\", \"collapse\", \"colonial\", \"comed\", \"comedk\", \"corona\", \"covid\", \"covid\", \"cut\", \"das\", \"death\", \"delhi\", \"dose\", \"drink\", \"duck\", \"exam\", \"facebook\", \"faith\", \"fall\", \"fall\", \"fall\", \"find\", \"flood\", \"force\", \"fuji\", \"galwan\", \"galwan_valley\", \"gandhi\", \"ganesh\", \"gdp\", \"growth\", \"gurugram\", \"housewife\", \"ibrahim\", \"imf\", \"indian\", \"indian\", \"internet\", \"israel\", \"japan\", \"kangana\", \"kapil\", \"kapil_mishra\", \"karnataka\", \"khan\", \"lashkar\", \"link\", \"link\", \"link\", \"links\", \"lk\", \"man\", \"man\", \"man\", \"mandir\", \"mercury\", \"message\", \"minister\", \"mishra\", \"modi\", \"modi\", \"modi_duck\", \"morph\", \"morphed\", \"mother\", \"municipality\", \"pakistan\", \"pakistan\", \"pandemic\", \"party\", \"pay\", \"pm\", \"pm\", \"pm_modi\", \"policy\", \"positive\", \"pr\", \"predict\", \"project\", \"protest\", \"protest_bangladesh\", \"ram\", \"red\", \"retirement\", \"rhea\", \"road\", \"road\", \"road\", \"rs\", \"rs\", \"screenshot\", \"sewage\", \"show\", \"show\", \"show\", \"show\", \"slogans\", \"slum\", \"spread\", \"student\", \"sydney\", \"teacher\", \"terrorist\", \"testing\", \"truck\", \"tsunami\", \"tweet\", \"uttar\", \"vaccine\", \"valley\", \"varanasi\", \"visual\", \"vp\", \"want\", \"west\", \"west_bengal\", \"wrongly\", \"zealand\"]}, \"R\": 10, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [1, 2, 3, 4, 5]};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el731249032592649826915758\", ldavis_el731249032592649826915758_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el731249032592649826915758\", ldavis_el731249032592649826915758_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el731249032592649826915758\", ldavis_el731249032592649826915758_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 803,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('wk36.json', 'r') as json_file:\n",
    "    dict_data = json.load(json_file)\n",
    "    viz_data = prepared_data_from_dict(dict_data)\n",
    "pyLDAvis.display(viz_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 804,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el731249219699048746164792\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el731249219699048746164792_data = {\"mdsDat\": {\"x\": [-0.15645124858314002, -0.19586192451177706, -0.10201763722168516, 0.4116588479581014, 0.04267196235850081], \"y\": [-0.11617502342190805, -0.2594353104057848, 0.3349724740505253, -0.09905561677979766, 0.13969347655696515], \"topics\": [1, 2, 3, 4, 5], \"cluster\": [1, 1, 1, 1, 1], \"Freq\": [23.726930815651663, 21.32335062678725, 18.302510887291522, 19.419184070359123, 17.228023599910447]}, \"tinfo\": {\"Term\": [\"flag\", \"khan\", \"covid\", \"new\", \"indian\", \"trailer\", \"get\", \"national\", \"go\", \"rss\", \"jason\", \"fast\", \"death\", \"bow\", \"tractor\", \"waterfall\", \"skull\", \"google\", \"vehicle\", \"morphed\", \"rss\", \"pm_modi\", \"modi\", \"pichai\", \"wave\", \"catch\", \"sundar\", \"sundar_pichai\", \"burqa\", \"police\", \"trump\", \"pm\", \"statue\", \"go\", \"flag\", \"putin\", \"zealand\", \"dirty\", \"full\", \"prey\", \"vaccine\", \"showcase\", \"malaysia\", \"bill\", \"dhoni\", \"covid\", \"new\", \"hyderabad\", \"link\", \"milk\", \"culture\", \"chowk\", \"uttar\", \"tricolour\", \"adulterate\", \"morph\", \"mumbai\", \"road\", \"bjp\", \"show\", \"man\", \"indian\", \"painting\", \"create\", \"ceremony\", \"next\", \"hostage\", \"queen\", \"survey\", \"ghatotkacha\", \"firework\", \"skeleton\", \"anthem\", \"prasad\", \"band\", \"rajendra\", \"mughal\", \"derogatory\", \"dr\", \"army\", \"anti\", \"mask\", \"youth\", \"russia\", \"krishna\", \"year\", \"go\", \"take\", \"niece\", \"anil\", \"jayate\", \"supreme\", \"reddy\", \"sadak\", \"cauvery\", \"upadhyay\", \"automaton\", \"prince\", \"khan\", \"get\", \"trailer\", \"singh\", \"make\", \"salman\", \"dislike\", \"mahesh\", \"bodyguard\", \"sushant\", \"mahesh_bhatt\", \"bhatt\", \"bahrain\", \"king\", \"ganapathy\", \"pact\", \"attack\", \"sweep\", \"reflect\", \"paint\", \"haryana\", \"colour\", \"greet\", \"bsp\", \"israeli\", \"independence_day\", \"egyptian\", \"burj\", \"national_flag\", \"street\", \"khalifa\", \"independence\", \"doval\", \"burj_khalifa\", \"flag\", \"national\", \"day\", \"man\"], \"Freq\": [12.0, 7.0, 6.0, 6.0, 5.0, 5.0, 5.0, 6.0, 8.0, 4.0, 0.9998300258146261, 0.9998300258146261, 1.9996600516292522, 0.9998300258146261, 0.9998300258146261, 0.9998300258146261, 0.9998300258146261, 1.9996600516292522, 0.9998300258146261, 2.9994900774438786, 4.99915012907313, 3.9993201032585044, 3.9993201032585044, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 4.99915012907313, 3.9993201032585044, 3.9993201032585044, 3.9993201032585044, 1.0001666841612114, 3.000500052483634, 1.0001666841612114, 1.0001666841612114, 1.0001666841612114, 3.000500052483634, 1.0001666841612114, 2.0003333683224227, 1.0001666841612114, 1.0001666841612114, 6.001000104967268, 6.001000104967268, 4.000666736644845, 4.000666736644845, 4.000666736644845, 2.0003333683224227, 2.0003333683224227, 2.0003333683224227, 2.0003333683224227, 2.0003333683224227, 2.0003333683224227, 4.000666736644845, 4.000666736644845, 3.000500052483634, 3.000500052483634, 3.000500052483634, 5.007770339995041, 3.0046622039970243, 3.0046622039970243, 3.0046622039970243, 1.0015540679990083, 1.0015540679990083, 1.0015540679990083, 1.0015540679990083, 1.0015540679990083, 4.006216271996033, 4.006216271996033, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 3.0046622039970243, 3.0046622039970243, 4.006216271996033, 3.0046622039970243, 2.000302882018691, 2.000302882018691, 1.0001514410093455, 1.0001514410093455, 1.0001514410093455, 1.0001514410093455, 1.0001514410093455, 2.000302882018691, 1.0001514410093455, 1.0001514410093455, 7.001060087065418, 5.000757205046728, 5.000757205046728, 4.000605764037382, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 1.9964239112837396, 0.9982119556418698, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 8.98390760077683, 4.991059778209349, 3.9928478225674793, 2.9946358669256097], \"Total\": [12.0, 7.0, 6.0, 6.0, 5.0, 5.0, 5.0, 6.0, 8.0, 4.0, 0.9998300258146261, 0.9998300258146261, 1.9996600516292522, 0.9998300258146261, 0.9998300258146261, 0.9998300258146261, 0.9998300258146261, 1.9996600516292522, 0.9998300258146261, 2.9994900774438786, 4.99915012907313, 3.9993201032585044, 3.9993201032585044, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 2.9994900774438786, 6.999483497395553, 5.000874171257513, 8.005536375254538, 12.983227704035334, 1.0001666841612114, 3.000500052483634, 1.0001666841612114, 1.0001666841612114, 1.0001666841612114, 3.000500052483634, 1.0001666841612114, 2.0003333683224227, 1.0001666841612114, 1.0001666841612114, 6.001000104967268, 6.001000104967268, 4.000666736644845, 4.000666736644845, 4.000666736644845, 2.0003333683224227, 2.0003333683224227, 2.0003333683224227, 2.0003333683224227, 2.0003333683224227, 2.0003333683224227, 5.000496762459472, 5.000818177654191, 4.00033007829826, 6.003438214296277, 8.994625996853122, 5.007770339995041, 3.0046622039970243, 3.0046622039970243, 3.0046622039970243, 1.0015540679990083, 1.0015540679990083, 1.0015540679990083, 1.0015540679990083, 1.0015540679990083, 4.006216271996033, 4.006216271996033, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 2.0031081359980165, 4.002874159638894, 4.0044922298116505, 8.005536375254538, 5.004658913972862, 2.000302882018691, 2.000302882018691, 1.0001514410093455, 1.0001514410093455, 1.0001514410093455, 1.0001514410093455, 1.0001514410093455, 2.000302882018691, 1.0001514410093455, 1.0001514410093455, 7.001060087065418, 5.000757205046728, 5.000757205046728, 4.000605764037382, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 3.0004543230280363, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 0.9982119556418698, 1.9964239112837396, 0.9982119556418698, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 2.9946358669256097, 12.983227704035334, 6.994167914207366, 4.993014506728691, 8.994625996853122], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\"], \"logprob\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -5.2311, -5.2311, -4.538, -5.2311, -5.2311, -5.2311, -5.2311, -4.538, -5.2311, -4.1325, -3.6217, -3.8448, -3.8448, -4.1325, -4.1325, -4.1325, -4.1325, -4.1325, -4.1325, -4.1325, -4.1325, -3.6217, -3.8448, -3.8448, -3.8448, -5.124, -4.0254, -5.124, -5.124, -5.124, -4.0254, -5.124, -4.4308, -5.124, -5.124, -3.3322, -3.3322, -3.7377, -3.7377, -3.7377, -4.4308, -4.4308, -4.4308, -4.4308, -4.4308, -4.4308, -3.7377, -3.7377, -4.0254, -4.0254, -4.0254, -3.3604, -3.8712, -3.8712, -3.8712, -4.9698, -4.9698, -4.9698, -4.9698, -4.9698, -3.5835, -3.5835, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -4.2767, -3.8712, -3.8712, -3.5835, -3.8712, -4.3373, -4.3373, -5.0304, -5.0304, -5.0304, -5.0304, -5.0304, -4.3373, -5.0304, -5.0304, -3.0845, -3.421, -3.421, -3.6441, -3.9318, -3.9318, -3.9318, -3.9318, -3.9318, -3.9318, -3.9318, -3.9318, -3.9318, -3.9318, -4.9127, -4.9127, -4.9127, -4.9127, -4.9127, -4.9127, -4.9127, -4.9127, -4.2195, -4.9127, -3.814, -3.814, -3.814, -3.814, -3.814, -3.814, -3.814, -3.814, -3.814, -3.814, -2.7154, -3.3032, -3.5264, -3.814], \"loglift\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.4386, 1.102, 1.2151, 0.7446, 0.261, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.5454, 1.3223, 1.3222, 1.2578, 0.8518, 0.4475, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.6981, 1.4113, 1.4109, 1.0058, 1.1879, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.6389, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.7586, 1.3904, 1.4212, 1.5351, 0.6588]}, \"token.table\": {\"Topic\": [2, 4, 3, 3, 3, 5, 4, 4, 3, 4, 2, 1, 2, 4, 1, 5, 5, 5, 1, 1, 4, 3, 2, 5, 2, 3, 2, 2, 5, 1, 3, 2, 2, 4, 5, 3, 5, 1, 3, 1, 5, 2, 5, 4, 3, 1, 3, 1, 5, 5, 3, 2, 5, 5, 3, 5, 1, 4, 5, 4, 4, 3, 5, 2, 4, 4, 4, 2, 1, 2, 5, 3, 2, 1, 2, 1, 3, 1, 2, 3, 5, 5, 2, 3, 4, 5, 5, 3, 1, 1, 2, 1, 1, 3, 2, 4, 2, 3, 3, 4, 5, 2, 4, 1, 3, 4, 4, 1, 2, 3, 2, 4, 3, 1, 1, 3, 5, 1, 1, 4, 3, 4, 5, 1, 2, 3, 1, 4, 2, 1, 4, 2, 2, 1, 1, 1, 1, 3, 3, 2], \"Freq\": [0.9998333436177679, 0.9998485819215611, 0.998448343380889, 0.998448343380889, 0.998448343380889, 1.0017912471875579, 0.9998485819215611, 0.9998485819215612, 0.998448343380889, 0.9998485819215612, 0.9998333436177679, 0.2499793718085883, 0.7499381154257649, 0.9998485819215612, 1.0001700030815093, 1.0017912471875579, 1.0017912471875579, 1.0017912471875579, 1.000170003081509, 1.000170003081509, 0.9998485819215611, 0.9984483433808891, 0.9998333436177679, 1.0017912471875579, 0.9998333436177679, 0.9984483433808891, 0.9998333436177679, 0.20027981065394043, 0.8011192426157617, 1.0001700030815093, 0.998448343380889, 0.9998333436177679, 0.9998333436177679, 0.9998485819215612, 1.0017912471875579, 0.998448343380889, 1.0017912471875579, 1.0001700030815093, 0.998448343380889, 0.3080897979442165, 0.6932020453744872, 0.9998333436177679, 1.0017912471875579, 0.999848581921561, 0.998448343380889, 0.49965421584544595, 0.49965421584544595, 1.0001700030815093, 1.0017912471875579, 1.0017912471875579, 0.998448343380889, 0.9998333436177679, 1.0017912471875579, 1.0017912471875579, 0.9984483433808891, 1.0017912471875579, 1.0001700030815093, 0.9998485819215611, 1.0017912471875579, 0.9998485819215612, 0.9998485819215612, 0.7494614820143721, 0.2498204940047907, 0.9998333436177679, 0.9998485819215612, 0.9998485819215612, 0.9998485819215612, 0.9998333436177679, 0.33353248940529445, 0.33353248940529445, 0.33353248940529445, 0.998448343380889, 0.9998333436177679, 1.0001700030815093, 0.9998333436177679, 1.000170003081509, 0.998448343380889, 0.19998013147560856, 0.7999205259024342, 0.28595252852556885, 0.714881321313922, 1.0017912471875579, 0.9998333436177679, 0.998448343380889, 0.9998485819215611, 1.0017912471875579, 1.0017912471875579, 0.9984483433808891, 1.000170003081509, 0.7143384225222413, 0.2857353690088965, 1.0001700030815093, 1.000170003081509, 0.998448343380889, 0.9998333436177679, 0.9998485819215611, 0.9998333436177679, 0.998448343380889, 0.998448343380889, 0.9998485819215611, 1.0017912471875579, 0.7998691129930944, 0.1999672782482736, 1.0001700030815093, 0.998448343380889, 0.9998485819215611, 0.9998485819215612, 0.16657121541097097, 0.4997136462329129, 0.33314243082194195, 0.9998333436177679, 0.9998485819215611, 0.998448343380889, 1.0001700030815093, 0.7998601570481358, 0.19996503926203396, 1.0017912471875579, 1.000170003081509, 1.000170003081509, 0.9998485819215611, 0.998448343380889, 0.9998485819215612, 1.0017912471875579, 0.19981381692327305, 0.19981381692327305, 0.5994414507698191, 1.0001700030815093, 0.999848581921561, 0.9998333436177679, 1.000170003081509, 0.9998485819215611, 0.9998333436177679, 0.9998333436177679, 1.0001700030815093, 1.0001700030815093, 1.000170003081509, 0.2497195505975634, 0.7491586517926903, 0.998448343380889, 0.9998333436177679], \"Term\": [\"adulterate\", \"anil\", \"anthem\", \"anti\", \"army\", \"attack\", \"automaton\", \"bahrain\", \"band\", \"bhatt\", \"bill\", \"bjp\", \"bjp\", \"bodyguard\", \"bow\", \"bsp\", \"burj\", \"burj_khalifa\", \"burqa\", \"catch\", \"cauvery\", \"ceremony\", \"chowk\", \"colour\", \"covid\", \"create\", \"culture\", \"day\", \"day\", \"death\", \"derogatory\", \"dhoni\", \"dirty\", \"dislike\", \"doval\", \"dr\", \"egyptian\", \"fast\", \"firework\", \"flag\", \"flag\", \"full\", \"ganapathy\", \"get\", \"ghatotkacha\", \"go\", \"go\", \"google\", \"greet\", \"haryana\", \"hostage\", \"hyderabad\", \"independence\", \"independence_day\", \"indian\", \"israeli\", \"jason\", \"jayate\", \"khalifa\", \"khan\", \"king\", \"krishna\", \"krishna\", \"link\", \"mahesh\", \"mahesh_bhatt\", \"make\", \"malaysia\", \"man\", \"man\", \"man\", \"mask\", \"milk\", \"modi\", \"morph\", \"morphed\", \"mughal\", \"mumbai\", \"mumbai\", \"national\", \"national\", \"national_flag\", \"new\", \"next\", \"niece\", \"pact\", \"paint\", \"painting\", \"pichai\", \"pm\", \"pm\", \"pm_modi\", \"police\", \"prasad\", \"prey\", \"prince\", \"putin\", \"queen\", \"rajendra\", \"reddy\", \"reflect\", \"road\", \"road\", \"rss\", \"russia\", \"sadak\", \"salman\", \"show\", \"show\", \"show\", \"showcase\", \"singh\", \"skeleton\", \"skull\", \"statue\", \"statue\", \"street\", \"sundar\", \"sundar_pichai\", \"supreme\", \"survey\", \"sushant\", \"sweep\", \"take\", \"take\", \"take\", \"tractor\", \"trailer\", \"tricolour\", \"trump\", \"upadhyay\", \"uttar\", \"vaccine\", \"vehicle\", \"waterfall\", \"wave\", \"year\", \"year\", \"youth\", \"zealand\"]}, \"R\": 10, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [1, 2, 3, 4, 5]};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el731249219699048746164792\", ldavis_el731249219699048746164792_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el731249219699048746164792\", ldavis_el731249219699048746164792_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el731249219699048746164792\", ldavis_el731249219699048746164792_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 804,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('wk35.json', 'r') as json_file:\n",
    "    dict_data = json.load(json_file)\n",
    "    viz_data = prepared_data_from_dict(dict_data)\n",
    "pyLDAvis.display(viz_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 805,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el731249219157603294483082\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el731249219157603294483082_data = {\"mdsDat\": {\"x\": [0.40222833681594894, -0.015541443783709457, -0.14410248403616435, -0.24258440899607536], \"y\": [-0.03513034934396969, 0.2541674999712354, -0.35594801631397904, 0.13691086568671343], \"topics\": [1, 2, 3, 4], \"cluster\": [1, 1, 1, 1], \"Freq\": [19.250328600476966, 30.35845822198089, 25.038750106859318, 25.352463070682823]}, \"tinfo\": {\"Term\": [\"president\", \"pm\", \"bengaluru\", \"delhi\", \"modi\", \"violence\", \"russian_president\", \"russian\", \"daughter\", \"putin\", \"recommend\", \"islam\", \"waqf\", \"publish\", \"bangalore\", \"hang\", \"fly\", \"hindustan\", \"hindustan_time\", \"officer\", \"violence\", \"times\", \"bengaluru_violence\", \"cable\", \"child\", \"hospital\", \"government\", \"school\", \"pilot\", \"swarajya\", \"rioter\", \"bengaluru\", \"unverified\", \"uttarakhand\", \"link\", \"attribute\", \"give\", \"prayagraj\", \"terrorist\", \"wayanad\", \"death\", \"hostage\", \"bjp\", \"arvind\", \"related\", \"delhi\", \"hoist\", \"flood\", \"chowk\", \"lal\", \"srinagar\", \"mumbai\", \"probe\", \"communal\", \"clash\", \"police\", \"flag\", \"show\", \"kashmir\", \"take\", \"man\", \"owaisi\", \"sooraj\", \"rajdeep\", \"next\", \"blame\", \"aaj\", \"ayodya\", \"sculpture\", \"riya\", \"former_president\", \"president\", \"russian_president\", \"putin\", \"daughter\", \"russian\", \"mukherjee\", \"bhoomi_pujan\", \"young\", \"pujan\", \"pranab_mukherjee\", \"pranab\", \"bhoomi\", \"peta\", \"accuse\", \"billboard\", \"movie\", \"cheetahs\", \"shinzo\", \"hyderabad\", \"yogi\", \"motorcyclist\", \"include\", \"pm\", \"modi\", \"congress\", \"minister\", \"south_africa\", \"south\", \"pm_modi\", \"africa\", \"king\", \"robot\", \"prime\", \"temple\", \"horrific\", \"fall\"], \"Freq\": [6.0, 6.0, 6.0, 5.0, 5.0, 4.0, 3.0, 3.0, 3.0, 3.0, 1.000078046805267, 1.000078046805267, 1.000078046805267, 2.000156093610534, 1.000078046805267, 2.000156093610534, 1.000078046805267, 3.0002341404158006, 3.0002341404158006, 1.000078046805267, 4.000312187221068, 3.0002341404158006, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 5.000390234026334, 2.000156093610534, 2.000156093610534, 2.000156093610534, 0.9999512785487521, 0.9999512785487521, 0.9999512785487521, 0.9999512785487521, 0.9999512785487521, 1.9999025570975042, 0.9999512785487521, 1.9999025570975042, 0.9999512785487521, 1.9999025570975042, 5.999707671292513, 3.9998051141950084, 3.9998051141950084, 3.9998051141950084, 3.9998051141950084, 2.9998538356462565, 2.9998538356462565, 1.9999025570975042, 1.9999025570975042, 1.9999025570975042, 1.9999025570975042, 5.999707671292513, 3.9998051141950084, 1.9999025570975042, 2.9998538356462565, 2.9998538356462565, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 2.999955247178083, 6.99989557674886, 3.9999403295707774, 3.9999403295707774, 3.9999403295707774, 3.9999403295707774, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.000027642242756, 1.000013821121378, 2.000027642242756, 1.000013821121378, 2.000027642242756, 1.000013821121378, 2.000027642242756, 2.000027642242756, 1.000013821121378, 1.000013821121378, 6.000082926728268, 5.00006910560689, 4.000055284485512, 3.000041463364134, 3.000041463364134, 3.000041463364134, 3.000041463364134, 3.000041463364134, 2.000027642242756, 2.000027642242756, 2.000027642242756, 4.000055284485512, 2.000027642242756, 2.000027642242756], \"Total\": [6.0, 6.0, 6.0, 5.0, 5.0, 4.0, 3.0, 3.0, 3.0, 3.0, 1.000078046805267, 1.000078046805267, 1.000078046805267, 2.000156093610534, 1.000078046805267, 2.000156093610534, 1.000078046805267, 3.0002341404158006, 3.0002341404158006, 1.000078046805267, 4.000312187221068, 3.0002341404158006, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 2.000156093610534, 6.000341512575086, 2.000156093610534, 2.000156093610534, 3.000141176003228, 0.9999512785487521, 0.9999512785487521, 0.9999512785487521, 0.9999512785487521, 0.9999512785487521, 1.9999025570975042, 0.9999512785487521, 1.9999025570975042, 0.9999512785487521, 1.9999025570975042, 5.999707671292513, 3.9998051141950084, 3.9998051141950084, 3.9998051141950084, 3.9998051141950084, 2.9998538356462565, 2.9998538356462565, 1.9999025570975042, 1.9999025570975042, 1.9999025570975042, 1.9999025570975042, 7.999677836077902, 4.9997901965877025, 1.9999025570975042, 3.9998676567676346, 5.999866560281707, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 0.9999850823926943, 2.999955247178083, 6.99989557674886, 3.9999403295707774, 3.9999403295707774, 3.9999403295707774, 3.9999403295707774, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.999955247178083, 2.000027642242756, 1.000013821121378, 2.000027642242756, 1.000013821121378, 2.000027642242756, 1.000013821121378, 2.000027642242756, 2.000027642242756, 1.000013821121378, 1.000013821121378, 6.000082926728268, 5.00006910560689, 4.000055284485512, 3.000041463364134, 3.000041463364134, 3.000041463364134, 3.000041463364134, 3.000041463364134, 2.000027642242756, 2.000027642242756, 2.000027642242756, 5.000040366878206, 2.000027642242756, 3.00001272463545], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\"], \"logprob\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -4.8122, -4.8122, -4.8122, -4.119, -4.8122, -4.119, -4.8122, -3.7136, -3.7136, -4.8122, -3.4259, -3.7136, -4.119, -4.119, -4.119, -4.119, -4.119, -4.119, -4.119, -4.119, -4.119, -3.2027, -4.119, -4.119, -4.119, -5.2679, -5.2679, -5.2679, -5.2679, -5.2679, -4.5747, -5.2679, -4.5747, -5.2679, -4.5747, -3.4761, -3.8816, -3.8816, -3.8816, -3.8816, -4.1692, -4.1692, -4.5747, -4.5747, -4.5747, -4.5747, -3.4761, -3.8816, -4.5747, -4.1692, -4.1692, -5.0752, -5.0752, -5.0752, -5.0752, -5.0752, -5.0752, -5.0752, -5.0752, -5.0752, -3.9766, -3.1293, -3.6889, -3.6889, -3.6889, -3.6889, -3.9766, -3.9766, -3.9766, -3.9766, -3.9766, -3.9766, -3.9766, -4.3944, -5.0876, -4.3944, -5.0876, -4.3944, -5.0876, -4.3944, -4.3944, -5.0876, -5.0876, -3.2958, -3.4782, -3.7013, -3.989, -3.989, -3.989, -3.989, -3.989, -4.3944, -4.3944, -4.3944, -3.7013, -4.3944, -4.3944], \"loglift\": [10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.6476, 1.4653, 1.6476, 1.6476, 1.2422, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 1.1921, 0.9044, 0.9689, 1.1921, 0.9044, 0.4989, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3847, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.3723, 1.1492, 1.3723, 0.9668]}, \"token.table\": {\"Topic\": [3, 4, 4, 2, 2, 3, 1, 1, 2, 1, 3, 3, 4, 2, 3, 1, 4, 1, 2, 2, 2, 4, 3, 2, 2, 3, 4, 2, 3, 2, 1, 3, 2, 1, 1, 1, 1, 2, 4, 1, 2, 4, 4, 1, 2, 4, 2, 1, 3, 2, 3, 4, 4, 4, 4, 4, 3, 2, 3, 1, 3, 4, 1, 4, 4, 2, 3, 3, 2, 3, 4, 2, 1, 3, 3, 3, 1, 2, 1, 3, 4, 3, 3, 1, 3, 4, 2, 3, 3, 4, 4, 2, 1, 2, 4, 3, 4, 2, 1, 1, 1, 1, 1, 2, 4, 3], \"Freq\": [1.000014917829844, 0.9999861790696427, 0.9999861790696427, 1.0000487238251434, 1.0000487238251434, 1.000014917829844, 0.9999219592855614, 0.8332859037308723, 0.16665718074617447, 0.9999219592855614, 1.000014917829844, 1.000014917829844, 0.9999861790696427, 1.0000487238251434, 1.000014917829844, 0.9999219592855614, 0.9999861790696427, 0.9999219592855614, 1.0000487238251434, 1.0000487238251434, 1.0000487238251434, 0.9999861790696427, 1.000014917829844, 1.0000487238251434, 1.0000487238251432, 0.33333191949094687, 0.6666638389818937, 0.75003020408403, 0.25001006802800996, 1.0000487238251434, 0.9999219592855614, 1.000014917829844, 1.0000487238251434, 0.9999219592855614, 0.9999219592855614, 0.9999219592855616, 0.9999219592855616, 1.0000487238251434, 0.9999861790696427, 0.9999219592855614, 1.0000487238251434, 0.9999861790696427, 0.9999861790696427, 0.9999219592855614, 1.0000487238251434, 0.9999861790696427, 1.0000487238251434, 0.6666352956977809, 0.33331764784889045, 0.5000111202238376, 0.16667037340794588, 0.33334074681589176, 0.9999861790696427, 0.9999861790696427, 0.9999861790696427, 0.9999861790696427, 1.000014917829844, 1.0000487238251432, 1.000014917829844, 0.9999219592855614, 1.000014917829844, 0.9999861790696427, 0.9999219592855614, 0.9999861790696427, 0.9999861790696427, 1.0000487238251434, 1.000014917829844, 1.000014917829844, 1.0000487238251434, 1.000014917829844, 0.9999861790696427, 1.0000487238251434, 0.9999219592855614, 1.000014917829844, 1.000014917829844, 1.000014917829844, 0.9999219592855614, 1.0000487238251434, 0.9999219592855614, 1.000014917829844, 0.9999861790696427, 1.000014917829844, 1.000014917829844, 0.9999219592855614, 1.000014917829844, 0.9999861790696427, 0.8000335699545859, 0.20000839248864646, 1.000014917829844, 0.9999861790696427, 0.9999861790696427, 1.0000487238251432, 0.9999219592855614, 0.7500248151770987, 0.25000827172569956, 0.1999983853379075, 0.79999354135163, 1.0000487238251434, 0.9999219592855616, 0.9999219592855614, 0.9999219592855614, 0.9999219592855614, 0.9999219592855614, 1.0000487238251434, 0.9999861790696427, 1.000014917829844], \"Term\": [\"aaj\", \"accuse\", \"africa\", \"arvind\", \"attribute\", \"ayodya\", \"bangalore\", \"bengaluru\", \"bengaluru\", \"bengaluru_violence\", \"bhoomi\", \"bhoomi_pujan\", \"billboard\", \"bjp\", \"blame\", \"cable\", \"cheetahs\", \"child\", \"chowk\", \"clash\", \"communal\", \"congress\", \"daughter\", \"death\", \"delhi\", \"fall\", \"fall\", \"flag\", \"flag\", \"flood\", \"fly\", \"former_president\", \"give\", \"government\", \"hang\", \"hindustan\", \"hindustan_time\", \"hoist\", \"horrific\", \"hospital\", \"hostage\", \"hyderabad\", \"include\", \"islam\", \"kashmir\", \"king\", \"lal\", \"link\", \"link\", \"man\", \"man\", \"man\", \"minister\", \"modi\", \"motorcyclist\", \"movie\", \"mukherjee\", \"mumbai\", \"next\", \"officer\", \"owaisi\", \"peta\", \"pilot\", \"pm\", \"pm_modi\", \"police\", \"pranab\", \"pranab_mukherjee\", \"prayagraj\", \"president\", \"prime\", \"probe\", \"publish\", \"pujan\", \"putin\", \"rajdeep\", \"recommend\", \"related\", \"rioter\", \"riya\", \"robot\", \"russian\", \"russian_president\", \"school\", \"sculpture\", \"shinzo\", \"show\", \"show\", \"sooraj\", \"south\", \"south_africa\", \"srinagar\", \"swarajya\", \"take\", \"take\", \"temple\", \"temple\", \"terrorist\", \"times\", \"unverified\", \"uttarakhand\", \"violence\", \"waqf\", \"wayanad\", \"yogi\", \"young\"]}, \"R\": 10, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [1, 2, 3, 4]};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el731249219157603294483082\", ldavis_el731249219157603294483082_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el731249219157603294483082\", ldavis_el731249219157603294483082_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el731249219157603294483082\", ldavis_el731249219157603294483082_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 805,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('wk34.json', 'r') as json_file:\n",
    "    dict_data = json.load(json_file)\n",
    "    viz_data = prepared_data_from_dict(dict_data)\n",
    "pyLDAvis.display(viz_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}