BERT_LV1_freezed-pad50_+tokens.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bf556963",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:17.586027Z",
     "iopub.status.busy": "2024-08-20T10:44:17.585252Z",
     "iopub.status.idle": "2024-08-20T10:44:18.240847Z",
     "shell.execute_reply": "2024-08-20T10:44:18.239969Z"
    },
    "papermill": {
     "duration": 0.671824,
     "end_time": "2024-08-20T10:44:18.242973",
     "exception": false,
     "start_time": "2024-08-20T10:44:17.571149",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/dataset-tachygraphy/Tachygraphy_dataset_main.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7427f0a8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:18.269137Z",
     "iopub.status.busy": "2024-08-20T10:44:18.268795Z",
     "iopub.status.idle": "2024-08-20T10:44:21.795248Z",
     "shell.execute_reply": "2024-08-20T10:44:21.794036Z"
    },
    "papermill": {
     "duration": 3.542167,
     "end_time": "2024-08-20T10:44:21.797762",
     "exception": false,
     "start_time": "2024-08-20T10:44:18.255595",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8c50a1f2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:21.824160Z",
     "iopub.status.busy": "2024-08-20T10:44:21.823736Z",
     "iopub.status.idle": "2024-08-20T10:44:23.411314Z",
     "shell.execute_reply": "2024-08-20T10:44:23.410568Z"
    },
    "papermill": {
     "duration": 1.603023,
     "end_time": "2024-08-20T10:44:23.413607",
     "exception": false,
     "start_time": "2024-08-20T10:44:21.810584",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b16e71ee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.441180Z",
     "iopub.status.busy": "2024-08-20T10:44:23.440900Z",
     "iopub.status.idle": "2024-08-20T10:44:23.541628Z",
     "shell.execute_reply": "2024-08-20T10:44:23.540750Z"
    },
    "papermill": {
     "duration": 0.11607,
     "end_time": "2024-08-20T10:44:23.543629",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.427559",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /usr/share/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e85fdbbc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.569920Z",
     "iopub.status.busy": "2024-08-20T10:44:23.569643Z",
     "iopub.status.idle": "2024-08-20T10:44:23.613097Z",
     "shell.execute_reply": "2024-08-20T10:44:23.612267Z"
    },
    "papermill": {
     "duration": 0.058854,
     "end_time": "2024-08-20T10:44:23.615049",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.556195",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4958, 3)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"/kaggle/input/dataset-tachygraphy/Tachygraphy_dataset_main.csv\")\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "112f6ab5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.641124Z",
     "iopub.status.busy": "2024-08-20T10:44:23.640879Z",
     "iopub.status.idle": "2024-08-20T10:44:23.655458Z",
     "shell.execute_reply": "2024-08-20T10:44:23.654585Z"
    },
    "papermill": {
     "duration": 0.029915,
     "end_time": "2024-08-20T10:44:23.657378",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.627463",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Meaning</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Last session of the day http://twitpic.com/67ezh</td>\n",
       "      <td>Last session of the day http://twitpic.com/67ezh</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shanghai is also really exciting (precisely --...</td>\n",
       "      <td>Shanghai is also really exciting (precisely --...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>submit the report ASAP!</td>\n",
       "      <td>submit the report as soon as possilbe</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy bday!</td>\n",
       "      <td>Happy Birthday!</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The OGs - I like it!!</td>\n",
       "      <td>The original gangsters - i like it!</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  \\\n",
       "0   Last session of the day http://twitpic.com/67ezh   \n",
       "1  Shanghai is also really exciting (precisely --...   \n",
       "2                            submit the report ASAP!   \n",
       "3                                        happy bday!   \n",
       "4                              The OGs - I like it!!   \n",
       "\n",
       "                                             Meaning Sentiment  \n",
       "0   Last session of the day http://twitpic.com/67ezh   neutral  \n",
       "1  Shanghai is also really exciting (precisely --...  positive  \n",
       "2              submit the report as soon as possilbe  negative  \n",
       "3                                    Happy Birthday!  positive  \n",
       "4                The original gangsters - i like it!  positive  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "269c85d6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.685792Z",
     "iopub.status.busy": "2024-08-20T10:44:23.685098Z",
     "iopub.status.idle": "2024-08-20T10:44:23.689104Z",
     "shell.execute_reply": "2024-08-20T10:44:23.688146Z"
    },
    "papermill": {
     "duration": 0.019644,
     "end_time": "2024-08-20T10:44:23.691076",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.671432",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# df.drop(columns=['Sentiment.1'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a0a45c04",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.718829Z",
     "iopub.status.busy": "2024-08-20T10:44:23.718473Z",
     "iopub.status.idle": "2024-08-20T10:44:23.726582Z",
     "shell.execute_reply": "2024-08-20T10:44:23.725741Z"
    },
    "papermill": {
     "duration": 0.024372,
     "end_time": "2024-08-20T10:44:23.728698",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.704326",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df=df.drop('Meaning',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6ad2ebb2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.755009Z",
     "iopub.status.busy": "2024-08-20T10:44:23.754746Z",
     "iopub.status.idle": "2024-08-20T10:44:23.759822Z",
     "shell.execute_reply": "2024-08-20T10:44:23.758922Z"
    },
    "papermill": {
     "duration": 0.020487,
     "end_time": "2024-08-20T10:44:23.761816",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.741329",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4958, 2)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "13ac4c36",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.791920Z",
     "iopub.status.busy": "2024-08-20T10:44:23.791325Z",
     "iopub.status.idle": "2024-08-20T10:44:23.796280Z",
     "shell.execute_reply": "2024-08-20T10:44:23.795337Z"
    },
    "papermill": {
     "duration": 0.022447,
     "end_time": "2024-08-20T10:44:23.798282",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.775835",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.rename(columns={'Text': 'tweet'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e387c73c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.826224Z",
     "iopub.status.busy": "2024-08-20T10:44:23.825554Z",
     "iopub.status.idle": "2024-08-20T10:44:23.830331Z",
     "shell.execute_reply": "2024-08-20T10:44:23.829513Z"
    },
    "papermill": {
     "duration": 0.02074,
     "end_time": "2024-08-20T10:44:23.832138",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.811398",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.rename(columns={'Sentiment': 'class'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5d645462",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.859731Z",
     "iopub.status.busy": "2024-08-20T10:44:23.859442Z",
     "iopub.status.idle": "2024-08-20T10:44:23.875171Z",
     "shell.execute_reply": "2024-08-20T10:44:23.874320Z"
    },
    "papermill": {
     "duration": 0.032039,
     "end_time": "2024-08-20T10:44:23.877160",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.845121",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "class\n",
       "neutral     1743\n",
       "positive    1643\n",
       "negative    1572\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['class'] = df['class'].str.lower()\n",
    "sentiment_counts = df['class'].value_counts()\n",
    "sentiment_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "94ae5600",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.905008Z",
     "iopub.status.busy": "2024-08-20T10:44:23.904456Z",
     "iopub.status.idle": "2024-08-20T10:44:23.910740Z",
     "shell.execute_reply": "2024-08-20T10:44:23.909994Z"
    },
    "papermill": {
     "duration": 0.022063,
     "end_time": "2024-08-20T10:44:23.912619",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.890556",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df[df['class'] != \"neutral\"]  ###droping neutral columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "83e291e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.940982Z",
     "iopub.status.busy": "2024-08-20T10:44:23.940711Z",
     "iopub.status.idle": "2024-08-20T10:44:23.947202Z",
     "shell.execute_reply": "2024-08-20T10:44:23.946314Z"
    },
    "papermill": {
     "duration": 0.023413,
     "end_time": "2024-08-20T10:44:23.949364",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.925951",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "label_encoder = LabelEncoder()\n",
    "label_encoder.fit(df['class'])\n",
    "encoded_labels = label_encoder.transform(df['class'])\n",
    "df['class'] = encoded_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "57b294e8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:23.976152Z",
     "iopub.status.busy": "2024-08-20T10:44:23.975918Z",
     "iopub.status.idle": "2024-08-20T10:44:23.984066Z",
     "shell.execute_reply": "2024-08-20T10:44:23.983246Z"
    },
    "papermill": {
     "duration": 0.023951,
     "end_time": "2024-08-20T10:44:23.986019",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.962068",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shanghai is also really exciting (precisely --...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>submit the report ASAP!</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy bday!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The OGs - I like it!!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>that`s great!! weee!! visitors!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  class\n",
       "1  Shanghai is also really exciting (precisely --...      1\n",
       "2                            submit the report ASAP!      0\n",
       "3                                        happy bday!      1\n",
       "4                              The OGs - I like it!!      1\n",
       "5                    that`s great!! weee!! visitors!      1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "269973a2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:24.013447Z",
     "iopub.status.busy": "2024-08-20T10:44:24.013189Z",
     "iopub.status.idle": "2024-08-20T10:44:24.021279Z",
     "shell.execute_reply": "2024-08-20T10:44:24.020192Z"
    },
    "papermill": {
     "duration": 0.023826,
     "end_time": "2024-08-20T10:44:24.023165",
     "exception": false,
     "start_time": "2024-08-20T10:44:23.999339",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tweet    1\n",
       "class    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e8f1d136",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:24.051029Z",
     "iopub.status.busy": "2024-08-20T10:44:24.050765Z",
     "iopub.status.idle": "2024-08-20T10:44:24.055195Z",
     "shell.execute_reply": "2024-08-20T10:44:24.054384Z"
    },
    "papermill": {
     "duration": 0.020503,
     "end_time": "2024-08-20T10:44:24.056965",
     "exception": false,
     "start_time": "2024-08-20T10:44:24.036462",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df=df.fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "968234e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:24.084440Z",
     "iopub.status.busy": "2024-08-20T10:44:24.084185Z",
     "iopub.status.idle": "2024-08-20T10:44:24.092377Z",
     "shell.execute_reply": "2024-08-20T10:44:24.091572Z"
    },
    "papermill": {
     "duration": 0.023934,
     "end_time": "2024-08-20T10:44:24.094308",
     "exception": false,
     "start_time": "2024-08-20T10:44:24.070374",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shanghai is also really exciting (precisely --...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>submit the report ASAP!</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy bday!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The OGs - I like it!!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>that`s great!! weee!! visitors!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  class\n",
       "1  Shanghai is also really exciting (precisely --...      1\n",
       "2                            submit the report ASAP!      0\n",
       "3                                        happy bday!      1\n",
       "4                              The OGs - I like it!!      1\n",
       "5                    that`s great!! weee!! visitors!      1"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d3e8c50c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:24.122319Z",
     "iopub.status.busy": "2024-08-20T10:44:24.122083Z",
     "iopub.status.idle": "2024-08-20T10:44:24.298342Z",
     "shell.execute_reply": "2024-08-20T10:44:24.297434Z"
    },
    "papermill": {
     "duration": 0.19283,
     "end_time": "2024-08-20T10:44:24.300563",
     "exception": false,
     "start_time": "2024-08-20T10:44:24.107733",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def text_cleaning(text):\n",
    "    text = str(text)\n",
    "    soup = BeautifulSoup(text, \"html.parser\")    ###removing html tages\n",
    "    text = re.sub(r'\\[[^]]*\\]', '', soup.get_text())      ##removing text within square brackets\n",
    "    pattern = r\"[^a-zA-Z0-9\\s,']\"                  # Removing unwanted characters\n",
    "    text = re.sub(pattern, '', text)\n",
    "    url_pattern = r'http\\S+|www\\S+'            ###removing urls\n",
    "    text = re.sub(url_pattern, '', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "06e04325",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:24.330087Z",
     "iopub.status.busy": "2024-08-20T10:44:24.329682Z",
     "iopub.status.idle": "2024-08-20T10:44:24.522714Z",
     "shell.execute_reply": "2024-08-20T10:44:24.521509Z"
    },
    "papermill": {
     "duration": 0.209631,
     "end_time": "2024-08-20T10:44:24.524765",
     "exception": false,
     "start_time": "2024-08-20T10:44:24.315134",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_23/2827669698.py:6: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  soup = BeautifulSoup(text, \"html.parser\")    ###removing html tages\n"
     ]
    }
   ],
   "source": [
    "df['tweet'] = df['tweet'].apply(text_cleaning).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e1176eef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:24.553434Z",
     "iopub.status.busy": "2024-08-20T10:44:24.553160Z",
     "iopub.status.idle": "2024-08-20T10:44:24.562807Z",
     "shell.execute_reply": "2024-08-20T10:44:24.561972Z"
    },
    "papermill": {
     "duration": 0.025973,
     "end_time": "2024-08-20T10:44:24.564648",
     "exception": false,
     "start_time": "2024-08-20T10:44:24.538675",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shanghai is also really exciting precisely  sk...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>submit the report ASAP</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy bday</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The OGs  I like it</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>thats great weee visitors</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>I THINK EVERYONE HATES ME ON HERE lol</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>soooooo wish i could, but im in school and mys...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>My bike was put on holdshould have known that ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Im in VA for the weekend, my youngest son turn...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Its coming out the socket I feel like my phone...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                tweet  class\n",
       "1   Shanghai is also really exciting precisely  sk...      1\n",
       "2                              submit the report ASAP      0\n",
       "3                                          happy bday      1\n",
       "4                                  The OGs  I like it      1\n",
       "5                           thats great weee visitors      1\n",
       "6               I THINK EVERYONE HATES ME ON HERE lol      0\n",
       "7   soooooo wish i could, but im in school and mys...      0\n",
       "10  My bike was put on holdshould have known that ...      0\n",
       "13  Im in VA for the weekend, my youngest son turn...      0\n",
       "14  Its coming out the socket I feel like my phone...      0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e4674362",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:24.635459Z",
     "iopub.status.busy": "2024-08-20T10:44:24.634914Z",
     "iopub.status.idle": "2024-08-20T10:44:24.642592Z",
     "shell.execute_reply": "2024-08-20T10:44:24.641761Z"
    },
    "papermill": {
     "duration": 0.065732,
     "end_time": "2024-08-20T10:44:24.644400",
     "exception": false,
     "start_time": "2024-08-20T10:44:24.578668",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lens=[len(i.split()) for i in df['tweet']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1ddbfce3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:24.673574Z",
     "iopub.status.busy": "2024-08-20T10:44:24.673286Z",
     "iopub.status.idle": "2024-08-20T10:44:24.977361Z",
     "shell.execute_reply": "2024-08-20T10:44:24.976486Z"
    },
    "papermill": {
     "duration": 0.321308,
     "end_time": "2024-08-20T10:44:24.979547",
     "exception": false,
     "start_time": "2024-08-20T10:44:24.658239",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([135., 493., 641., 585., 429., 438., 237., 174.,  74.,   9.]),\n",
       " array([ 0. ,  3.2,  6.4,  9.6, 12.8, 16. , 19.2, 22.4, 25.6, 28.8, 32. ]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkEElEQVR4nO3df3DU9Z3H8VdCkuXnbgyQXXIkEKsVUgFr0LDV2lZyBJo6eMQ76aU27TEwpYlXSKWSDoKlHcPQnlo6CP114s1Jabk59IQBxVBCT5YoUUYETcGLF7ywiZXLLsTmB8nn/mj5XleCuJDk+0nyfMx8Z8j3+9nd937nO5PnbHaXBGOMEQAAgEUS3R4AAADgwwgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANZJcnuAK9Hd3a3GxkaNGTNGCQkJbo8DAAA+BmOMzp49q4yMDCUmfvRrJAMyUBobG5WZmen2GAAA4AqcOnVKEydO/Mg1AzJQxowZI+lPT9Dr9bo8DQAA+Dii0agyMzOd3+MfZUAGyoU/63i9XgIFAIAB5uO8PYM3yQIAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDpJbg+AoWvyyl1ujxC3d9YVuj0CAAwJvIICAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOvEHSj/8z//o6985SsaO3asRowYoWnTpunw4cPOcWOMVq9erQkTJmjEiBHKz8/XiRMnYu7jzJkzKi4ultfrVWpqqhYtWqRz585d/bMBAACDQlyB8r//+7+67bbblJycrN27d+v48eP6p3/6J11zzTXOmvXr12vDhg3avHmzampqNGrUKBUUFKitrc1ZU1xcrGPHjmnv3r3auXOnDhw4oCVLlvTeswIAAANagjHGfNzFK1eu1EsvvaTf/e53PR43xigjI0Pf/va39cADD0iSIpGI/H6/tmzZooULF+rNN99UTk6OXnnlFc2cOVOStGfPHn3xi1/Uu+++q4yMjMvOEY1G5fP5FIlE5PV6P+74sMzklbvcHiFu76wrdHsEABiw4vn9HdcrKP/xH/+hmTNn6m//9m+Vnp6uT3/60/r5z3/uHK+vr1c4HFZ+fr6zz+fzKS8vT6FQSJIUCoWUmprqxIkk5efnKzExUTU1NT0+bnt7u6LRaMwGAAAGr7gC5b/+67+0adMmXX/99Xr++ee1dOlS/eM//qOeeuopSVI4HJYk+f3+mNv5/X7nWDgcVnp6eszxpKQkpaWlOWs+rLKyUj6fz9kyMzPjGRsAAAwwcQVKd3e3br75Zj3yyCP69Kc/rSVLlmjx4sXavHlzX80nSaqoqFAkEnG2U6dO9enjAQAAd8UVKBMmTFBOTk7MvqlTp6qhoUGSFAgEJElNTU0xa5qampxjgUBAzc3NMcfPnz+vM2fOOGs+zOPxyOv1xmwAAGDwiitQbrvtNtXV1cXs+/3vf69JkyZJkrKzsxUIBFRVVeUcj0ajqqmpUTAYlCQFg0G1tLSotrbWWbNv3z51d3crLy/vip8IAAAYPJLiWbx8+XJ95jOf0SOPPKK/+7u/08svv6yf/exn+tnPfiZJSkhI0LJly/SDH/xA119/vbKzs/XQQw8pIyNDd999t6Q/veIyd+5c509DnZ2dKisr08KFCz/WJ3gAAMDgF1eg3HLLLdqxY4cqKiq0du1aZWdn6/HHH1dxcbGz5jvf+Y5aW1u1ZMkStbS06Pbbb9eePXs0fPhwZ83TTz+tsrIyzZ49W4mJiSoqKtKGDRt671kBAIABLa7vQbEF34MyOPA9KAAwtPTZ96AAAAD0h7j+xAMMdbzqAwD9g1dQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGCduALl4YcfVkJCQsw2ZcoU53hbW5tKS0s1duxYjR49WkVFRWpqaoq5j4aGBhUWFmrkyJFKT0/XihUrdP78+d55NgAAYFBIivcGn/rUp/Tiiy/+/x0k/f9dLF++XLt27dL27dvl8/lUVlamBQsW6KWXXpIkdXV1qbCwUIFAQAcPHtTp06f11a9+VcnJyXrkkUd64ekAAIDBIO5ASUpKUiAQuGh/JBLRL3/5S23dulV33nmnJOnJJ5/U1KlTdejQIc2aNUsvvPCCjh8/rhdffFF+v1833XSTvv/97+vBBx/Uww8/rJSUlKt/RgAAYMCL+z0oJ06cUEZGhq699loVFxeroaFBklRbW6vOzk7l5+c7a6dMmaKsrCyFQiFJUigU0rRp0+T3+501BQUFikajOnbs2CUfs729XdFoNGYDAACDV1yBkpeXpy1btmjPnj3atGmT6uvr9dnPflZnz55VOBxWSkqKUlNTY27j9/sVDoclSeFwOCZOLhy/cOxSKisr5fP5nC0zMzOesQEAwAAT15945s2b5/x7+vTpysvL06RJk/Sb3/xGI0aM6PXhLqioqFB5ebnzczQaJVIAABjErupjxqmpqfrkJz+pkydPKhAIqKOjQy0tLTFrmpqanPesBAKBiz7Vc+Hnnt7XcoHH45HX643ZAADA4HVVgXLu3Dm9/fbbmjBhgnJzc5WcnKyqqirneF1dnRoaGhQMBiVJwWBQR48eVXNzs7Nm79698nq9ysnJuZpRAADAIBLXn3geeOAB3XXXXZo0aZIaGxu1Zs0aDRs2TF/+8pfl8/m0aNEilZeXKy0tTV6vV/fff7+CwaBmzZolSZozZ45ycnJ03333af369QqHw1q1apVKS0vl8Xj65AkOFZNX7nJ7BAAAek1cgfLuu+/qy1/+st5//32NHz9et99+uw4dOqTx48dLkh577DElJiaqqKhI7e3tKigo0BNPPOHcftiwYdq5c6eWLl2qYDCoUaNGqaSkRGvXru3dZwUAAAa0BGOMcXuIeEWjUfl8PkUiEd6P8me8goJLeWddodsjAICk+H5/83/xAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwTpLbAwDAh01eucvtEeL2zrpCt0cABhVeQQEAANYhUAAAgHUIFAAAYB0CBQAAWIc3yQKD3EB8wykA8AoKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArHNVgbJu3TolJCRo2bJlzr62tjaVlpZq7NixGj16tIqKitTU1BRzu4aGBhUWFmrkyJFKT0/XihUrdP78+asZBQAADCJXHCivvPKKfvrTn2r69Okx+5cvX67nnntO27dvV3V1tRobG7VgwQLneFdXlwoLC9XR0aGDBw/qqaee0pYtW7R69eorfxYAAGBQuaJAOXfunIqLi/Xzn/9c11xzjbM/Eonol7/8pR599FHdeeedys3N1ZNPPqmDBw/q0KFDkqQXXnhBx48f17/+67/qpptu0rx58/T9739fGzduVEdHR+88KwAAMKBdUaCUlpaqsLBQ+fn5Mftra2vV2dkZs3/KlCnKyspSKBSSJIVCIU2bNk1+v99ZU1BQoGg0qmPHjl3JOAAAYJBJivcG27Zt06uvvqpXXnnlomPhcFgpKSlKTU2N2e/3+xUOh501fxknF45fONaT9vZ2tbe3Oz9Ho9F4xwYAAANIXK+gnDp1St/61rf09NNPa/jw4X0100UqKyvl8/mcLTMzs98eGwAA9L+4AqW2tlbNzc26+eablZSUpKSkJFVXV2vDhg1KSkqS3+9XR0eHWlpaYm7X1NSkQCAgSQoEAhd9qufCzxfWfFhFRYUikYiznTp1Kp6xAQDAABNXoMyePVtHjx7VkSNHnG3mzJkqLi52/p2cnKyqqirnNnV1dWpoaFAwGJQkBYNBHT16VM3Nzc6avXv3yuv1Kicnp8fH9Xg88nq9MRsAABi84noPypgxY3TjjTfG7Bs1apTGjh3r7F+0aJHKy8uVlpYmr9er+++/X8FgULNmzZIkzZkzRzk5Obrvvvu0fv16hcNhrVq1SqWlpfJ4PL30tAAAwEAW95tkL+exxx5TYmKiioqK1N7eroKCAj3xxBPO8WHDhmnnzp1aunSpgsGgRo0apZKSEq1du7a3RwEAAANUgjHGuD1EvKLRqHw+nyKRCH/u+bPJK3e5PQIwpL2zrtDtEQDrxfP7m/+LBwAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGCduAJl06ZNmj59urxer7xer4LBoHbv3u0cb2trU2lpqcaOHavRo0erqKhITU1NMffR0NCgwsJCjRw5Uunp6VqxYoXOnz/fO88GAAAMCnEFysSJE7Vu3TrV1tbq8OHDuvPOOzV//nwdO3ZMkrR8+XI999xz2r59u6qrq9XY2KgFCxY4t+/q6lJhYaE6Ojp08OBBPfXUU9qyZYtWr17du88KAAAMaAnGGHM1d5CWlqYf/vCHuueeezR+/Hht3bpV99xzjyTprbfe0tSpUxUKhTRr1izt3r1bX/rSl9TY2Ci/3y9J2rx5sx588EG99957SklJ+ViPGY1G5fP5FIlE5PV6r2b8QWPyyl1ujwAMae+sK3R7BMB68fz+vuL3oHR1dWnbtm1qbW1VMBhUbW2tOjs7lZ+f76yZMmWKsrKyFAqFJEmhUEjTpk1z4kSSCgoKFI1GnVdhAAAAkuK9wdGjRxUMBtXW1qbRo0drx44dysnJ0ZEjR5SSkqLU1NSY9X6/X+FwWJIUDodj4uTC8QvHLqW9vV3t7e3Oz9FoNN6xAQDAABL3Kyg33HCDjhw5opqaGi1dulQlJSU6fvx4X8zmqKyslM/nc7bMzMw+fTwAAOCuuAMlJSVF1113nXJzc1VZWakZM2boxz/+sQKBgDo6OtTS0hKzvqmpSYFAQJIUCAQu+lTPhZ8vrOlJRUWFIpGIs506dSresQEAwABy1d+D0t3drfb2duXm5io5OVlVVVXOsbq6OjU0NCgYDEqSgsGgjh49qubmZmfN3r175fV6lZOTc8nH8Hg8zkebL2wAAGDwius9KBUVFZo3b56ysrJ09uxZbd26Vfv379fzzz8vn8+nRYsWqby8XGlpafJ6vbr//vsVDAY1a9YsSdKcOXOUk5Oj++67T+vXr1c4HNaqVatUWloqj8fTJ08QAAAMPHEFSnNzs7761a/q9OnT8vl8mj59up5//nn99V//tSTpscceU2JiooqKitTe3q6CggI98cQTzu2HDRumnTt3aunSpQoGgxo1apRKSkq0du3a3n1WAABgQLvq70FxA9+DcjG+BwVwF9+DAlxev3wPCgAAQF8hUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWCfJ7QEAYDCYvHKX2yPE7Z11hW6PAFwSr6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALBOXIFSWVmpW265RWPGjFF6erruvvtu1dXVxaxpa2tTaWmpxo4dq9GjR6uoqEhNTU0xaxoaGlRYWKiRI0cqPT1dK1as0Pnz56/+2QAAgEEhrkCprq5WaWmpDh06pL1796qzs1Nz5sxRa2urs2b58uV67rnntH37dlVXV6uxsVELFixwjnd1damwsFAdHR06ePCgnnrqKW3ZskWrV6/uvWcFAAAGtARjjLnSG7/33ntKT09XdXW17rjjDkUiEY0fP15bt27VPffcI0l66623NHXqVIVCIc2aNUu7d+/Wl770JTU2Nsrv90uSNm/erAcffFDvvfeeUlJSLvu40WhUPp9PkUhEXq/3SscfVCav3OX2CAAGmHfWFbo9AoaYeH5/X9V7UCKRiCQpLS1NklRbW6vOzk7l5+c7a6ZMmaKsrCyFQiFJUigU0rRp05w4kaSCggJFo1EdO3asx8dpb29XNBqN2QAAwOB1xYHS3d2tZcuW6bbbbtONN94oSQqHw0pJSVFqamrMWr/fr3A47Kz5yzi5cPzCsZ5UVlbK5/M5W2Zm5pWODQAABoArDpTS0lK98cYb2rZtW2/O06OKigpFIhFnO3XqVJ8/JgAAcE/SldyorKxMO3fu1IEDBzRx4kRnfyAQUEdHh1paWmJeRWlqalIgEHDWvPzyyzH3d+FTPhfWfJjH45HH47mSUQEAwAAU1ysoxhiVlZVpx44d2rdvn7Kzs2OO5+bmKjk5WVVVVc6+uro6NTQ0KBgMSpKCwaCOHj2q5uZmZ83evXvl9XqVk5NzNc8FAAAMEnG9glJaWqqtW7fq2Wef1ZgxY5z3jPh8Po0YMUI+n0+LFi1SeXm50tLS5PV6df/99ysYDGrWrFmSpDlz5ignJ0f33Xef1q9fr3A4rFWrVqm0tJRXSQAAgKQ4A2XTpk2SpM9//vMx+5988kl97WtfkyQ99thjSkxMVFFRkdrb21VQUKAnnnjCWTts2DDt3LlTS5cuVTAY1KhRo1RSUqK1a9de3TMBAACDxlV9D4pb+B6Ui/E9KADixfegoL/12/egAAAA9AUCBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1rug/CwQADHwD8Qse+XK5oYNXUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWSXJ7ABtNXrnL7REAABjSeAUFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWiTtQDhw4oLvuuksZGRlKSEjQM888E3PcGKPVq1drwoQJGjFihPLz83XixImYNWfOnFFxcbG8Xq9SU1O1aNEinTt37qqeCAAAGDziDpTW1lbNmDFDGzdu7PH4+vXrtWHDBm3evFk1NTUaNWqUCgoK1NbW5qwpLi7WsWPHtHfvXu3cuVMHDhzQkiVLrvxZAACAQSUp3hvMmzdP8+bN6/GYMUaPP/64Vq1apfnz50uS/uVf/kV+v1/PPPOMFi5cqDfffFN79uzRK6+8opkzZ0qSfvKTn+iLX/yifvSjHykjI+Mqng4AABgMevU9KPX19QqHw8rPz3f2+Xw+5eXlKRQKSZJCoZBSU1OdOJGk/Px8JSYmqqampsf7bW9vVzQajdkAAMDg1auBEg6HJUl+vz9mv9/vd46Fw2Glp6fHHE9KSlJaWpqz5sMqKyvl8/mcLTMzszfHBgAAlhkQn+KpqKhQJBJxtlOnTrk9EgAA6EO9GiiBQECS1NTUFLO/qanJORYIBNTc3Bxz/Pz58zpz5oyz5sM8Ho+8Xm/MBgAABq9eDZTs7GwFAgFVVVU5+6LRqGpqahQMBiVJwWBQLS0tqq2tddbs27dP3d3dysvL681xAADAABX3p3jOnTunkydPOj/X19fryJEjSktLU1ZWlpYtW6Yf/OAHuv7665Wdna2HHnpIGRkZuvvuuyVJU6dO1dy5c7V48WJt3rxZnZ2dKisr08KFC/kEDwAAkHQFgXL48GF94QtfcH4uLy+XJJWUlGjLli36zne+o9bWVi1ZskQtLS26/fbbtWfPHg0fPty5zdNPP62ysjLNnj1biYmJKioq0oYNG3rh6QAAgMEgwRhj3B4iXtFoVD6fT5FIpE/ejzJ55a5ev08AwNV7Z12h2yPgKsTz+3tAfIoHAAAMLQQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOvE/U2yAAC4ZSB+kSZfLndleAUFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1klyewAAAAazySt3uT3CFXlnXaGrj+/qKygbN27U5MmTNXz4cOXl5enll192cxwAAGAJ1wLl17/+tcrLy7VmzRq9+uqrmjFjhgoKCtTc3OzWSAAAwBKuBcqjjz6qxYsX6+tf/7pycnK0efNmjRw5Uv/8z//s1kgAAMASrrwHpaOjQ7W1taqoqHD2JSYmKj8/X6FQ6KL17e3tam9vd36ORCKSpGg02ifzdbd/0Cf3CwDAQNEXv2Mv3Kcx5rJrXQmUP/zhD+rq6pLf74/Z7/f79dZbb120vrKyUt/73vcu2p+ZmdlnMwIAMJT5Hu+7+z579qx8Pt9HrhkQn+KpqKhQeXm583N3d7fOnDmjsWPHKiEhoVcfKxqNKjMzU6dOnZLX6+3V+x7IOC+XxrnpGeelZ5yXS+Pc9GwwnRdjjM6ePauMjIzLrnUlUMaNG6dhw4apqakpZn9TU5MCgcBF6z0ejzweT8y+1NTUvhxRXq93wF8IfYHzcmmcm55xXnrGebk0zk3PBst5udwrJxe48ibZlJQU5ebmqqqqytnX3d2tqqoqBYNBN0YCAAAWce1PPOXl5SopKdHMmTN166236vHHH1dra6u+/vWvuzUSAACwhGuBcu+99+q9997T6tWrFQ6HddNNN2nPnj0XvXG2v3k8Hq1Zs+aiPykNdZyXS+Pc9Izz0jPOy6Vxbno2VM9Lgvk4n/UBAADoR/xngQAAwDoECgAAsA6BAgAArEOgAAAA6xAof2Hjxo2aPHmyhg8frry8PL388stuj+S6hx9+WAkJCTHblClT3B6r3x04cEB33XWXMjIylJCQoGeeeSbmuDFGq1ev1oQJEzRixAjl5+frxIkT7gzbzy53br72ta9ddA3NnTvXnWH7UWVlpW655RaNGTNG6enpuvvuu1VXVxezpq2tTaWlpRo7dqxGjx6toqKii77AcrD5OOfl85///EXXzDe+8Q2XJu4/mzZt0vTp050vZAsGg9q9e7dzfKhdLwTKn/36179WeXm51qxZo1dffVUzZsxQQUGBmpub3R7NdZ/61Kd0+vRpZ/vP//xPt0fqd62trZoxY4Y2btzY4/H169drw4YN2rx5s2pqajRq1CgVFBSora2tnyftf5c7N5I0d+7cmGvoV7/6VT9O6I7q6mqVlpbq0KFD2rt3rzo7OzVnzhy1trY6a5YvX67nnntO27dvV3V1tRobG7VgwQIXp+57H+e8SNLixYtjrpn169e7NHH/mThxotatW6fa2lodPnxYd955p+bPn69jx45JGoLXi4Exxphbb73VlJaWOj93dXWZjIwMU1lZ6eJU7luzZo2ZMWOG22NYRZLZsWOH83N3d7cJBALmhz/8obOvpaXFeDwe86tf/cqFCd3z4XNjjDElJSVm/vz5rsxjk+bmZiPJVFdXG2P+dI0kJyeb7du3O2vefPNNI8mEQiG3xux3Hz4vxhjzuc99znzrW99ybyiLXHPNNeYXv/jFkLxeeAVFUkdHh2pra5Wfn+/sS0xMVH5+vkKhkIuT2eHEiRPKyMjQtddeq+LiYjU0NLg9klXq6+sVDodjrh+fz6e8vDyunz/bv3+/0tPTdcMNN2jp0qV6//333R6p30UiEUlSWlqaJKm2tladnZ0x182UKVOUlZU1pK6bD5+XC55++mmNGzdON954oyoqKvTBBx+4MZ5rurq6tG3bNrW2tioYDA7J62VA/G/Gfe0Pf/iDurq6LvoWW7/fr7feesulqeyQl5enLVu26IYbbtDp06f1ve99T5/97Gf1xhtvaMyYMW6PZ4VwOCxJPV4/F44NZXPnztWCBQuUnZ2tt99+W9/97nc1b948hUIhDRs2zO3x+kV3d7eWLVum2267TTfeeKOkP103KSkpF/3Hp0PpuunpvEjS3//932vSpEnKyMjQ66+/rgcffFB1dXX693//dxen7R9Hjx5VMBhUW1ubRo8erR07dignJ0dHjhwZctcLgYKPNG/ePOff06dPV15eniZNmqTf/OY3WrRokYuTYaBYuHCh8+9p06Zp+vTp+sQnPqH9+/dr9uzZLk7Wf0pLS/XGG28MyfdvfZRLnZclS5Y4/542bZomTJig2bNn6+2339YnPvGJ/h6zX91www06cuSIIpGI/u3f/k0lJSWqrq52eyxX8CceSePGjdOwYcMuejd0U1OTAoGAS1PZKTU1VZ/85Cd18uRJt0exxoVrhOvn47n22ms1bty4IXMNlZWVaefOnfrtb3+riRMnOvsDgYA6OjrU0tISs36oXDeXOi89ycvLk6Qhcc2kpKTouuuuU25uriorKzVjxgz9+Mc/HpLXC4GiP10Qubm5qqqqcvZ1d3erqqpKwWDQxcnsc+7cOb399tuaMGGC26NYIzs7W4FAIOb6iUajqqmp4frpwbvvvqv3339/0F9DxhiVlZVpx44d2rdvn7Kzs2OO5+bmKjk5Oea6qaurU0NDw6C+bi53Xnpy5MgRSRr010xPuru71d7ePjSvF7ffpWuLbdu2GY/HY7Zs2WKOHz9ulixZYlJTU004HHZ7NFd9+9vfNvv37zf19fXmpZdeMvn5+WbcuHGmubnZ7dH61dmzZ81rr71mXnvtNSPJPProo+a1114z//3f/22MMWbdunUmNTXVPPvss+b111838+fPN9nZ2eaPf/yjy5P3vY86N2fPnjUPPPCACYVCpr6+3rz44ovm5ptvNtdff71pa2tze/Q+tXTpUuPz+cz+/fvN6dOnne2DDz5w1nzjG98wWVlZZt++febw4cMmGAyaYDDo4tR973Ln5eTJk2bt2rXm8OHDpr6+3jz77LPm2muvNXfccYfLk/e9lStXmurqalNfX29ef/11s3LlSpOQkGBeeOEFY8zQu14IlL/wk5/8xGRlZZmUlBRz6623mkOHDrk9kuvuvfdeM2HCBJOSkmL+6q/+ytx7773m5MmTbo/V7377298aSRdtJSUlxpg/fdT4oYceMn6/33g8HjN79mxTV1fn7tD95KPOzQcffGDmzJljxo8fb5KTk82kSZPM4sWLh0T493ROJJknn3zSWfPHP/7RfPOb3zTXXHONGTlypPmbv/kbc/r0afeG7geXOy8NDQ3mjjvuMGlpacbj8ZjrrrvOrFixwkQiEXcH7wf/8A//YCZNmmRSUlLM+PHjzezZs504MWboXS8JxhjTf6/XAAAAXB7vQQEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFjn/wCE2vB4RlGAmgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.hist(lens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a068d4b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:25.009673Z",
     "iopub.status.busy": "2024-08-20T10:44:25.008912Z",
     "iopub.status.idle": "2024-08-20T10:44:25.942196Z",
     "shell.execute_reply": "2024-08-20T10:44:25.941399Z"
    },
    "papermill": {
     "duration": 0.95045,
     "end_time": "2024-08-20T10:44:25.944435",
     "exception": false,
     "start_time": "2024-08-20T10:44:24.993985",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoModel, BertTokenizerFast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ed6cca5f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:25.974540Z",
     "iopub.status.busy": "2024-08-20T10:44:25.974248Z",
     "iopub.status.idle": "2024-08-20T10:44:25.987177Z",
     "shell.execute_reply": "2024-08-20T10:44:25.986294Z"
    },
    "papermill": {
     "duration": 0.030571,
     "end_time": "2024-08-20T10:44:25.989605",
     "exception": false,
     "start_time": "2024-08-20T10:44:25.959034",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# split into train, validation and test sets in the ration 70 : 15 : 15\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_text, temp_text, train_labels, temp_labels = train_test_split(df['tweet'], df['class'], \n",
    "                                                                    random_state=2021, \n",
    "                                                                    test_size=0.2, \n",
    "                                                                    stratify=df['class'])\n",
    "\n",
    "\n",
    "val_text, test_text, val_labels, test_labels = train_test_split(temp_text, temp_labels, \n",
    "                                                                random_state=2021, \n",
    "                                                                test_size=0.5, \n",
    "                                                                stratify=temp_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f4c847f0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:26.021091Z",
     "iopub.status.busy": "2024-08-20T10:44:26.020808Z",
     "iopub.status.idle": "2024-08-20T10:44:29.838306Z",
     "shell.execute_reply": "2024-08-20T10:44:29.837313Z"
    },
    "papermill": {
     "duration": 3.835408,
     "end_time": "2024-08-20T10:44:29.840809",
     "exception": false,
     "start_time": "2024-08-20T10:44:26.005401",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7247b6cd205c4e52a83fbc246d894b94",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd8088a298464de78ed730d6fa518b56",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bert = AutoModel.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ac6bb0e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:29.872613Z",
     "iopub.status.busy": "2024-08-20T10:44:29.872124Z",
     "iopub.status.idle": "2024-08-20T10:44:30.600166Z",
     "shell.execute_reply": "2024-08-20T10:44:30.599109Z"
    },
    "papermill": {
     "duration": 0.746056,
     "end_time": "2024-08-20T10:44:30.602342",
     "exception": false,
     "start_time": "2024-08-20T10:44:29.856286",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2e5f47e4bb84c4c883d6fddb59356e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "651a2eea54e74e8a88a7285854196003",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e26b1e1beb8d449c8be2b8476b3ac1ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "07cde6af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:30.634068Z",
     "iopub.status.busy": "2024-08-20T10:44:30.633794Z",
     "iopub.status.idle": "2024-08-20T10:44:30.840660Z",
     "shell.execute_reply": "2024-08-20T10:44:30.839715Z"
    },
    "papermill": {
     "duration": 0.224925,
     "end_time": "2024-08-20T10:44:30.842600",
     "exception": false,
     "start_time": "2024-08-20T10:44:30.617675",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([192., 469., 502., 424., 315., 268., 194., 145.,  55.,   8.]),\n",
       " array([ 1. ,  4.1,  7.2, 10.3, 13.4, 16.5, 19.6, 22.7, 25.8, 28.9, 32. ]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAg4klEQVR4nO3de2xUdf7/8VdvM1xnaoHO0KXl4gWoXIxVy8TLutClstXgUhN0CVaXQGQHInRloRsExc2W4EYUw2Wz64KbFVE2iwYIKBYpWRluVSKCNEDqFlOmRU1noNoL7fn9sb/OfkeQOlB6Pm2fj+Qk9Jwz0/ccT9Knp2emcZZlWQIAADBIvN0DAAAAfB+BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4iXYPcDVaWlpUVVWlvn37Ki4uzu5xAADAj2BZls6fP6+0tDTFx1/5GkmnDJSqqiqlp6fbPQYAALgKZ86c0aBBg664T6cMlL59+0r67wt0uVw2TwMAAH6McDis9PT0yM/xK+mUgdL6ax2Xy0WgAADQyfyY2zO4SRYAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGCcmALlueeeU1xcXNQyYsSIyPb6+nr5/X7169dPffr0UX5+vqqrq6Oeo7KyUnl5eerVq5dSU1O1YMECXbx4sX1eDQAA6BJi/mvGt956qz744IP/PUHi/55i/vz52r59uzZv3iy32605c+ZoypQp+uijjyRJzc3NysvLk9fr1b59+3T27Fk9/vjjSkpK0h//+Md2eDkAAKAriDlQEhMT5fV6L1kfCoX02muvaePGjRo/frwkaf369Ro5cqT279+vcePG6f3339fx48f1wQcfyOPx6LbbbtMLL7yghQsX6rnnnpPD4bj2V4ROY8ii7XaPELMvlufZPQIAdAsx34Ny8uRJpaWladiwYZo2bZoqKyslSWVlZWpqalJOTk5k3xEjRigjI0OBQECSFAgENHr0aHk8nsg+ubm5CofDOnbs2A9+z4aGBoXD4agFAAB0XTEFSnZ2tjZs2KCdO3dq7dq1qqio0L333qvz588rGAzK4XAoOTk56jEej0fBYFCSFAwGo+KkdXvrth9SXFwst9sdWdLT02MZGwAAdDIx/Ypn0qRJkX+PGTNG2dnZGjx4sN5++2317Nmz3YdrVVRUpMLCwsjX4XCYSAEAoAu7prcZJycn65ZbbtGpU6fk9XrV2Nio2traqH2qq6sj96x4vd5L3tXT+vXl7mtp5XQ65XK5ohYAANB1XVOgXLhwQadPn9bAgQOVlZWlpKQklZSURLaXl5ersrJSPp9PkuTz+XT06FHV1NRE9tm1a5dcLpcyMzOvZRQAANCFxPQrnmeeeUYPPfSQBg8erKqqKi1dulQJCQl67LHH5Ha7NWPGDBUWFiolJUUul0tz586Vz+fTuHHjJEkTJ05UZmampk+frhUrVigYDGrx4sXy+/1yOp3X5QUCAIDOJ6ZA+fLLL/XYY4/p66+/1oABA3TPPfdo//79GjBggCRp5cqVio+PV35+vhoaGpSbm6s1a9ZEHp+QkKBt27Zp9uzZ8vl86t27twoKCrRs2bL2fVUAAKBTi7Msy7J7iFiFw2G53W6FQiHuR+nE+BwUAOheYvn5zd/iAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcRLtHgDtY8ii7XaPAABAu+EKCgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjJNo9ANCZDFm03e4RYvbF8jy7RwCAmHEFBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYJxrCpTly5crLi5O8+bNi6yrr6+X3+9Xv3791KdPH+Xn56u6ujrqcZWVlcrLy1OvXr2UmpqqBQsW6OLFi9cyCgAA6EKuOlAOHTqkP//5zxozZkzU+vnz52vr1q3avHmzSktLVVVVpSlTpkS2Nzc3Ky8vT42Njdq3b59ef/11bdiwQUuWLLn6VwEAALqUqwqUCxcuaNq0afrLX/6iG264IbI+FArptdde00svvaTx48crKytL69ev1759+7R//35J0vvvv6/jx4/rH//4h2677TZNmjRJL7zwglavXq3Gxsb2eVUAAKBTu6pA8fv9ysvLU05OTtT6srIyNTU1Ra0fMWKEMjIyFAgEJEmBQECjR4+Wx+OJ7JObm6twOKxjx45d9vs1NDQoHA5HLQAAoOtKjPUBmzZt0scff6xDhw5dsi0YDMrhcCg5OTlqvcfjUTAYjOzzf+OkdXvrtsspLi7W888/H+uoAACgk4rpCsqZM2f09NNP64033lCPHj2u10yXKCoqUigUiixnzpzpsO8NAAA6XkyBUlZWppqaGt1+++1KTExUYmKiSktLtWrVKiUmJsrj8aixsVG1tbVRj6uurpbX65Ukeb3eS97V0/p16z7f53Q65XK5ohYAANB1xRQoEyZM0NGjR3XkyJHIcscdd2jatGmRfyclJamkpCTymPLyclVWVsrn80mSfD6fjh49qpqamsg+u3btksvlUmZmZju9LAAA0JnFdA9K3759NWrUqKh1vXv3Vr9+/SLrZ8yYocLCQqWkpMjlcmnu3Lny+XwaN26cJGnixInKzMzU9OnTtWLFCgWDQS1evFh+v19Op7OdXhYAAOjMYr5Jti0rV65UfHy88vPz1dDQoNzcXK1ZsyayPSEhQdu2bdPs2bPl8/nUu3dvFRQUaNmyZe09CgAA6KTiLMuy7B4iVuFwWG63W6FQiPtR/r8hi7bbPQIM9cXyPLtHAABJsf385m/xAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMk2j3AACuryGLtts9Qsy+WJ5n9wgAbMYVFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYJ6ZAWbt2rcaMGSOXyyWXyyWfz6cdO3ZEttfX18vv96tfv37q06eP8vPzVV1dHfUclZWVysvLU69evZSamqoFCxbo4sWL7fNqAABAlxBToAwaNEjLly9XWVmZDh8+rPHjx2vy5Mk6duyYJGn+/PnaunWrNm/erNLSUlVVVWnKlCmRxzc3NysvL0+NjY3at2+fXn/9dW3YsEFLlixp31cFAAA6tTjLsqxreYKUlBS9+OKLeuSRRzRgwABt3LhRjzzyiCTpxIkTGjlypAKBgMaNG6cdO3bowQcfVFVVlTwejyRp3bp1Wrhwoc6dOyeHw/Gjvmc4HJbb7VYoFJLL5bqW8buMzvhhXMAP4YPagK4plp/fV30PSnNzszZt2qS6ujr5fD6VlZWpqalJOTk5kX1GjBihjIwMBQIBSVIgENDo0aMjcSJJubm5CofDkaswl9PQ0KBwOBy1AACArivmQDl69Kj69Okjp9Opp556Slu2bFFmZqaCwaAcDoeSk5Oj9vd4PAoGg5KkYDAYFSet21u3/ZDi4mK53e7Ikp6eHuvYAACgE4k5UIYPH64jR47owIEDmj17tgoKCnT8+PHrMVtEUVGRQqFQZDlz5sx1/X4AAMBeMf+xQIfDoZtuukmSlJWVpUOHDumVV17R1KlT1djYqNra2qirKNXV1fJ6vZIkr9ergwcPRj1f67t8Wve5HKfTKafTGeuoAACgk7rmz0FpaWlRQ0ODsrKylJSUpJKSksi28vJyVVZWyufzSZJ8Pp+OHj2qmpqayD67du2Sy+VSZmbmtY4CAAC6iJiuoBQVFWnSpEnKyMjQ+fPntXHjRu3Zs0fvvfee3G63ZsyYocLCQqWkpMjlcmnu3Lny+XwaN26cJGnixInKzMzU9OnTtWLFCgWDQS1evFh+v58rJAAAICKmQKmpqdHjjz+us2fPyu12a8yYMXrvvff085//XJK0cuVKxcfHKz8/Xw0NDcrNzdWaNWsij09ISNC2bds0e/Zs+Xw+9e7dWwUFBVq2bFn7vioAANCpXfPnoNiBz0G5FJ+Dgq6Ez0EBuqYO+RwUAACA64VAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcRLtHgAAvm/Iou12jxCzL5bn2T0C0KVwBQUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcWIKlOLiYt15553q27evUlNT9fDDD6u8vDxqn/r6evn9fvXr1099+vRRfn6+qquro/aprKxUXl6eevXqpdTUVC1YsEAXL1689lcDAAC6hJgCpbS0VH6/X/v379euXbvU1NSkiRMnqq6uLrLP/PnztXXrVm3evFmlpaWqqqrSlClTItubm5uVl5enxsZG7du3T6+//ro2bNigJUuWtN+rAgAAnVqcZVnW1T743LlzSk1NVWlpqe677z6FQiENGDBAGzdu1COPPCJJOnHihEaOHKlAIKBx48Zpx44devDBB1VVVSWPxyNJWrdunRYuXKhz587J4XC0+X3D4bDcbrdCoZBcLtfVjt+lDFm03e4RgG7ti+V5do8AGC+Wn9/XdA9KKBSSJKWkpEiSysrK1NTUpJycnMg+I0aMUEZGhgKBgCQpEAho9OjRkTiRpNzcXIXDYR07duyy36ehoUHhcDhqAQAAXddVB0pLS4vmzZunu+++W6NGjZIkBYNBORwOJScnR+3r8XgUDAYj+/zfOGnd3rrtcoqLi+V2uyNLenr61Y4NAAA6gasOFL/fr88++0ybNm1qz3kuq6ioSKFQKLKcOXPmun9PAABgn8SredCcOXO0bds27d27V4MGDYqs93q9amxsVG1tbdRVlOrqanm93sg+Bw8ejHq+1nf5tO7zfU6nU06n82pGBQAAnVBMV1Asy9KcOXO0ZcsW7d69W0OHDo3anpWVpaSkJJWUlETWlZeXq7KyUj6fT5Lk8/l09OhR1dTURPbZtWuXXC6XMjMzr+W1AACALiKmKyh+v18bN27Uu+++q759+0buGXG73erZs6fcbrdmzJihwsJCpaSkyOVyae7cufL5fBo3bpwkaeLEicrMzNT06dO1YsUKBYNBLV68WH6/n6skADqtzvhOOt55BJPFFChr166VJN1///1R69evX68nnnhCkrRy5UrFx8crPz9fDQ0Nys3N1Zo1ayL7JiQkaNu2bZo9e7Z8Pp969+6tgoICLVu27NpeCQAA6DKu6XNQ7HK9PwelM/6fEADEiiso6Ggd9jkoAAAA1wOBAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4iXYPAACwx5BF2+0eIWZfLM+zewR0EK6gAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAODEHyt69e/XQQw8pLS1NcXFxeuedd6K2W5alJUuWaODAgerZs6dycnJ08uTJqH2++eYbTZs2TS6XS8nJyZoxY4YuXLhwTS8EAAB0HTEHSl1dncaOHavVq1dfdvuKFSu0atUqrVu3TgcOHFDv3r2Vm5ur+vr6yD7Tpk3TsWPHtGvXLm3btk179+7VrFmzrv5VAACALiUx1gdMmjRJkyZNuuw2y7L08ssva/HixZo8ebIk6e9//7s8Ho/eeecdPfroo/r888+1c+dOHTp0SHfccYck6dVXX9UvfvEL/elPf1JaWto1vBwAANAVtOs9KBUVFQoGg8rJyYmsc7vdys7OViAQkCQFAgElJydH4kSScnJyFB8frwMHDlz2eRsaGhQOh6MWAADQdbVroASDQUmSx+OJWu/xeCLbgsGgUlNTo7YnJiYqJSUlss/3FRcXy+12R5b09PT2HBsAABimU7yLp6ioSKFQKLKcOXPG7pEAAMB11K6B4vV6JUnV1dVR66urqyPbvF6vampqorZfvHhR33zzTWSf73M6nXK5XFELAADouto1UIYOHSqv16uSkpLIunA4rAMHDsjn80mSfD6famtrVVZWFtln9+7damlpUXZ2dnuOAwAAOqmY38Vz4cIFnTp1KvJ1RUWFjhw5opSUFGVkZGjevHn6wx/+oJtvvllDhw7Vs88+q7S0ND388MOSpJEjR+qBBx7QzJkztW7dOjU1NWnOnDl69NFHeQcPAACQdBWBcvjwYf3sZz+LfF1YWChJKigo0IYNG/S73/1OdXV1mjVrlmpra3XPPfdo586d6tGjR+Qxb7zxhubMmaMJEyYoPj5e+fn5WrVqVTu8HAAA0BXEWZZl2T1ErMLhsNxut0Kh0HW5H2XIou3t/pwAgGv3xfI8u0fANYjl53eneBcPAADoXggUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcRLtHgAAgB9ryKLtdo8Qsy+W59k9QqfEFRQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcRLtHgAAgK5syKLtdo9wVb5Ynmfr97f1Csrq1as1ZMgQ9ejRQ9nZ2Tp48KCd4wAAAEPYFihvvfWWCgsLtXTpUn388ccaO3ascnNzVVNTY9dIAADAELYFyksvvaSZM2fqySefVGZmptatW6devXrpb3/7m10jAQAAQ9hyD0pjY6PKyspUVFQUWRcfH6+cnBwFAoFL9m9oaFBDQ0Pk61AoJEkKh8PXZb6Whm+vy/MCANBZXI+fsa3PaVlWm/vaEihfffWVmpub5fF4otZ7PB6dOHHikv2Li4v1/PPPX7I+PT39us0IAEB35n75+j33+fPn5Xa7r7hPp3gXT1FRkQoLCyNft7S06JtvvlG/fv0UFxd3yf7hcFjp6ek6c+aMXC5XR47aaXCM2sYxujKOT9s4Rm3jGF1ZVzs+lmXp/PnzSktLa3NfWwKlf//+SkhIUHV1ddT66upqeb3eS/Z3Op1yOp1R65KTk9v8Pi6Xq0v8B72eOEZt4xhdGcenbRyjtnGMrqwrHZ+2rpy0suUmWYfDoaysLJWUlETWtbS0qKSkRD6fz46RAACAQWz7FU9hYaEKCgp0xx136K677tLLL7+suro6Pfnkk3aNBAAADGFboEydOlXnzp3TkiVLFAwGddttt2nnzp2X3Dh7NZxOp5YuXXrJr4XwPxyjtnGMrozj0zaOUds4RlfWnY9PnPVj3usDAADQgfhjgQAAwDgECgAAMA6BAgAAjEOgAAAA43TJQFm9erWGDBmiHj16KDs7WwcPHrR7JGM899xziouLi1pGjBhh91i22bt3rx566CGlpaUpLi5O77zzTtR2y7K0ZMkSDRw4UD179lROTo5Onjxpz7A2aesYPfHEE5ecUw888IA9w9qguLhYd955p/r27avU1FQ9/PDDKi8vj9qnvr5efr9f/fr1U58+fZSfn3/JB1V2ZT/mGN1///2XnEdPPfWUTRN3vLVr12rMmDGRD2Tz+XzasWNHZHt3PIe6XKC89dZbKiws1NKlS/Xxxx9r7Nixys3NVU1Njd2jGePWW2/V2bNnI8u///1vu0eyTV1dncaOHavVq1dfdvuKFSu0atUqrVu3TgcOHFDv3r2Vm5ur+vr6Dp7UPm0dI0l64IEHos6pN998swMntFdpaan8fr/279+vXbt2qampSRMnTlRdXV1kn/nz52vr1q3avHmzSktLVVVVpSlTptg4dcf6McdIkmbOnBl1Hq1YscKmiTveoEGDtHz5cpWVlenw4cMaP368Jk+erGPHjknqpueQ1cXcddddlt/vj3zd3NxspaWlWcXFxTZOZY6lS5daY8eOtXsMI0mytmzZEvm6paXF8nq91osvvhhZV1tbazmdTuvNN9+0YUL7ff8YWZZlFRQUWJMnT7ZlHhPV1NRYkqzS0lLLsv57ziQlJVmbN2+O7PP5559bkqxAIGDXmLb6/jGyLMv66U9/aj399NP2DWWgG264wfrrX//abc+hLnUFpbGxUWVlZcrJyYmsi4+PV05OjgKBgI2TmeXkyZNKS0vTsGHDNG3aNFVWVto9kpEqKioUDAajzie3263s7GzOp+/Zs2ePUlNTNXz4cM2ePVtff/213SPZJhQKSZJSUlIkSWVlZWpqaoo6j0aMGKGMjIxuex59/xi1euONN9S/f3+NGjVKRUVF+vbbb+0Yz3bNzc3atGmT6urq5PP5uu051Cn+mvGP9dVXX6m5ufmST6P1eDw6ceKETVOZJTs7Wxs2bNDw4cN19uxZPf/887r33nv12WefqW/fvnaPZ5RgMChJlz2fWrfhv7/emTJlioYOHarTp0/r97//vSZNmqRAIKCEhAS7x+tQLS0tmjdvnu6++26NGjVK0n/PI4fDcckfOO2u59HljpEk/epXv9LgwYOVlpamTz/9VAsXLlR5ebn+9a9/2Thtxzp69Kh8Pp/q6+vVp08fbdmyRZmZmTpy5Ei3PIe6VKCgbZMmTYr8e8yYMcrOztbgwYP19ttva8aMGTZOhs7q0Ucfjfx79OjRGjNmjG688Ubt2bNHEyZMsHGyjuf3+/XZZ5916/u62vJDx2jWrFmRf48ePVoDBw7UhAkTdPr0ad14440dPaYthg8friNHjigUCumf//ynCgoKVFpaavdYtulSv+Lp37+/EhISLrmzubq6Wl6v16apzJacnKxbbrlFp06dsnsU47SeM5xPsRk2bJj69+/f7c6pOXPmaNu2bfrwww81aNCgyHqv16vGxkbV1tZG7d8dz6MfOkaXk52dLUnd6jxyOBy66aablJWVpeLiYo0dO1avvPJKtz2HulSgOBwOZWVlqaSkJLKupaVFJSUl8vl8Nk5mrgsXLuj06dMaOHCg3aMYZ+jQofJ6vVHnUzgc1oEDBzifruDLL7/U119/3W3OKcuyNGfOHG3ZskW7d+/W0KFDo7ZnZWUpKSkp6jwqLy9XZWVltzmP2jpGl3PkyBFJ6jbn0eW0tLSooaGh+55Ddt+l2942bdpkOZ1Oa8OGDdbx48etWbNmWcnJyVYwGLR7NCP89re/tfbs2WNVVFRYH330kZWTk2P179/fqqmpsXs0W5w/f9765JNPrE8++cSSZL300kvWJ598Yv3nP/+xLMuyli9fbiUnJ1vvvvuu9emnn1qTJ0+2hg4dan333Xc2T95xrnSMzp8/bz3zzDNWIBCwKioqrA8++MC6/fbbrZtvvtmqr6+3e/QOMXv2bMvtdlt79uyxzp49G1m+/fbbyD5PPfWUlZGRYe3evds6fPiw5fP5LJ/PZ+PUHautY3Tq1Clr2bJl1uHDh62Kigrr3XfftYYNG2bdd999Nk/ecRYtWmSVlpZaFRUV1qeffmotWrTIiouLs95//33LsrrnOdTlAsWyLOvVV1+1MjIyLIfDYd11113W/v377R7JGFOnTrUGDhxoORwO6yc/+Yk1depU69SpU3aPZZsPP/zQknTJUlBQYFnWf99q/Oyzz1oej8dyOp3WhAkTrPLycnuH7mBXOkbffvutNXHiRGvAgAFWUlKSNXjwYGvmzJnd6n8ILndsJFnr16+P7PPdd99Zv/nNb6wbbrjB6tWrl/XLX/7SOnv2rH1Dd7C2jlFlZaV13333WSkpKZbT6bRuuukma8GCBVYoFLJ38A7061//2ho8eLDlcDisAQMGWBMmTIjEiWV1z3MozrIsq+Ou1wAAALStS92DAgAAugYCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHH+HyiwxoucFWB2AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_lens=[len(i.split()) for i in train_text]\n",
    "plt.hist(train_lens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d1737265",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:30.875080Z",
     "iopub.status.busy": "2024-08-20T10:44:30.874762Z",
     "iopub.status.idle": "2024-08-20T10:44:30.890214Z",
     "shell.execute_reply": "2024-08-20T10:44:30.889122Z"
    },
    "papermill": {
     "duration": 0.033873,
     "end_time": "2024-08-20T10:44:30.892088",
     "exception": false,
     "start_time": "2024-08-20T10:44:30.858215",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count    3215.000000\n",
      "mean       12.223328\n",
      "std         6.223879\n",
      "min         0.000000\n",
      "25%         7.000000\n",
      "50%        11.000000\n",
      "75%        17.000000\n",
      "max        32.000000\n",
      "Name: tweet, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "sequence_lengths = df['tweet'].apply(lambda x: len(x.split()))\n",
    "print(sequence_lengths.describe())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "60054a37",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:30.924071Z",
     "iopub.status.busy": "2024-08-20T10:44:30.923825Z",
     "iopub.status.idle": "2024-08-20T10:44:30.927539Z",
     "shell.execute_reply": "2024-08-20T10:44:30.926649Z"
    },
    "papermill": {
     "duration": 0.022263,
     "end_time": "2024-08-20T10:44:30.929670",
     "exception": false,
     "start_time": "2024-08-20T10:44:30.907407",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pad_len = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f7b87e76",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:30.961572Z",
     "iopub.status.busy": "2024-08-20T10:44:30.961271Z",
     "iopub.status.idle": "2024-08-20T10:44:31.096747Z",
     "shell.execute_reply": "2024-08-20T10:44:31.095855Z"
    },
    "papermill": {
     "duration": 0.153987,
     "end_time": "2024-08-20T10:44:31.098855",
     "exception": false,
     "start_time": "2024-08-20T10:44:30.944868",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2760: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# tokenizing and encoding sequences \n",
    "tokens_train = tokenizer.batch_encode_plus(\n",
    "    train_text.tolist(),\n",
    "    max_length = pad_len,\n",
    "    pad_to_max_length=True,\n",
    "    add_special_tokens=True,\n",
    "    truncation=True,\n",
    "    return_token_type_ids=True\n",
    ")\n",
    "\n",
    "tokens_val = tokenizer.batch_encode_plus(\n",
    "    val_text.tolist(),\n",
    "    max_length = pad_len,\n",
    "    pad_to_max_length=True,\n",
    "    add_special_tokens=True,\n",
    "    truncation=True,\n",
    "    return_token_type_ids=True\n",
    ")\n",
    "\n",
    "tokens_test = tokenizer.batch_encode_plus(\n",
    "    test_text.tolist(),\n",
    "    max_length = pad_len,\n",
    "    pad_to_max_length=True,\n",
    "    add_special_tokens=True,\n",
    "    truncation=True,\n",
    "    return_token_type_ids=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3c0e865b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:31.132320Z",
     "iopub.status.busy": "2024-08-20T10:44:31.132031Z",
     "iopub.status.idle": "2024-08-20T10:44:31.293976Z",
     "shell.execute_reply": "2024-08-20T10:44:31.293214Z"
    },
    "papermill": {
     "duration": 0.180794,
     "end_time": "2024-08-20T10:44:31.295897",
     "exception": false,
     "start_time": "2024-08-20T10:44:31.115103",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_seq = torch.tensor(tokens_train['input_ids'])\n",
    "train_mask = torch.tensor(tokens_train['attention_mask'])\n",
    "train_token_type_ids = torch.tensor(tokens_train[\"token_type_ids\"])\n",
    "train_y = torch.tensor(train_labels.tolist())\n",
    "\n",
    "val_seq = torch.tensor(tokens_val['input_ids'])\n",
    "val_mask = torch.tensor(tokens_val['attention_mask'])\n",
    "val_token_type_ids = torch.tensor(tokens_val[\"token_type_ids\"])\n",
    "val_y = torch.tensor(val_labels.tolist())\n",
    "\n",
    "test_seq = torch.tensor(tokens_test['input_ids'])\n",
    "test_mask = torch.tensor(tokens_test['attention_mask'])\n",
    "test_token_type_ids = torch.tensor(tokens_test[\"token_type_ids\"])\n",
    "test_y = torch.tensor(test_labels.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "f42d3c5b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:31.328794Z",
     "iopub.status.busy": "2024-08-20T10:44:31.328462Z",
     "iopub.status.idle": "2024-08-20T10:44:31.335432Z",
     "shell.execute_reply": "2024-08-20T10:44:31.334662Z"
    },
    "papermill": {
     "duration": 0.025455,
     "end_time": "2024-08-20T10:44:31.337329",
     "exception": false,
     "start_time": "2024-08-20T10:44:31.311874",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "batch_size = 64  ##batch size can be adjusted or modified\n",
    "train_data = TensorDataset(train_seq, train_mask, train_token_type_ids, train_y)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
    "val_data = TensorDataset(val_seq, val_mask, val_token_type_ids, val_y)\n",
    "val_sampler = SequentialSampler(val_data)\n",
    "val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "fb52b136",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:31.370104Z",
     "iopub.status.busy": "2024-08-20T10:44:31.369809Z",
     "iopub.status.idle": "2024-08-20T10:44:31.374890Z",
     "shell.execute_reply": "2024-08-20T10:44:31.373947Z"
    },
    "papermill": {
     "duration": 0.023651,
     "end_time": "2024-08-20T10:44:31.376712",
     "exception": false,
     "start_time": "2024-08-20T10:44:31.353061",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for param in bert.parameters():\n",
    "    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "85c4a87e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:31.409667Z",
     "iopub.status.busy": "2024-08-20T10:44:31.409311Z",
     "iopub.status.idle": "2024-08-20T10:44:31.420676Z",
     "shell.execute_reply": "2024-08-20T10:44:31.419697Z"
    },
    "papermill": {
     "duration": 0.03012,
     "end_time": "2024-08-20T10:44:31.422541",
     "exception": false,
     "start_time": "2024-08-20T10:44:31.392421",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "##model building\n",
    "class SentimentClassifier(nn.Module):\n",
    "     \n",
    "    def __init__(self, n_classes):\n",
    "        super(SentimentClassifier, self).__init__()\n",
    "        self.bert = BertModel.from_pretrained(MODEL_NAME)\n",
    "        self.drop = nn.Dropout(p=0.3)\n",
    "        self.out = nn.Linear(self.bert.config.hidden_size, n_classes)\n",
    "    \n",
    "    # Forward propagaion\n",
    "    def forward(self, input_ids, attention_mask, token_type_ids):\n",
    "        _, pooled_output = self.bert(\n",
    "          input_ids=input_ids,\n",
    "          attention_mask=attention_mask,\n",
    "            token_type_ids=token_type_ids\n",
    "        ) \n",
    "        output = self.drop(pooled_output)  ##drop out layer\n",
    "        return self.out(output)\n",
    "    \n",
    "class BERT_architecture(nn.Module):\n",
    "\n",
    "    def __init__(self, bert):\n",
    "        super(BERT_architecture, self).__init__()\n",
    "\n",
    "        self.bert = bert \n",
    "        \n",
    "        self.dropout = nn.Dropout(0.2) ##needs adjusment\n",
    "        self.relu = nn.ReLU() ##activation function\n",
    "        \n",
    "        self.fc1 = nn.Linear(768, 512) #dense layer\n",
    "        \n",
    "        # Dense layer 2 (Output layer)\n",
    "        self.fc2 = nn.Linear(512, 2)  #2 classes\n",
    "\n",
    "        # Softmax activation function\n",
    "        self.softmax = nn.LogSoftmax(dim=1)  \n",
    "\n",
    "    # Defining forward pass\n",
    "    def forward(self, sent_id, mask, token_type_ids): \n",
    "        _, cls_hs = self.bert(sent_id, attention_mask=mask, token_type_ids=token_type_ids, return_dict=False)\n",
    "        x = self.fc1(cls_hs)\n",
    "        x = self.relu(x)\n",
    "        x = self.dropout(x)\n",
    "        # Output layer\n",
    "        x = self.fc2(x)\n",
    "        x = self.softmax(x)\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "50966ac4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:31.455783Z",
     "iopub.status.busy": "2024-08-20T10:44:31.455502Z",
     "iopub.status.idle": "2024-08-20T10:44:31.467400Z",
     "shell.execute_reply": "2024-08-20T10:44:31.466398Z"
    },
    "papermill": {
     "duration": 0.030861,
     "end_time": "2024-08-20T10:44:31.469579",
     "exception": false,
     "start_time": "2024-08-20T10:44:31.438718",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = BERT_architecture(bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "72a448f6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:31.501746Z",
     "iopub.status.busy": "2024-08-20T10:44:31.501453Z",
     "iopub.status.idle": "2024-08-20T10:44:31.800247Z",
     "shell.execute_reply": "2024-08-20T10:44:31.799262Z"
    },
    "papermill": {
     "duration": 0.317461,
     "end_time": "2024-08-20T10:44:31.802337",
     "exception": false,
     "start_time": "2024-08-20T10:44:31.484876",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BERT_architecture(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSdpaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.2, inplace=False)\n",
       "  (relu): ReLU()\n",
       "  (fc1): Linear(in_features=768, out_features=512, bias=True)\n",
       "  (fc2): Linear(in_features=512, out_features=2, bias=True)\n",
       "  (softmax): LogSoftmax(dim=1)\n",
       ")"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_seq = test_seq.to(device)\n",
    "test_mask = test_mask.to(device)\n",
    "test_token_type_ids = test_token_type_ids.to(device)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8a351ac3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:31.835457Z",
     "iopub.status.busy": "2024-08-20T10:44:31.835194Z",
     "iopub.status.idle": "2024-08-20T10:44:32.450321Z",
     "shell.execute_reply": "2024-08-20T10:44:32.448815Z"
    },
    "papermill": {
     "duration": 0.635412,
     "end_time": "2024-08-20T10:44:32.453755",
     "exception": false,
     "start_time": "2024-08-20T10:44:31.818343",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import AdamW\n",
    "optimizer = AdamW(model.parameters(),lr = 1e-3)  ## learning rate needs to be adjusted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4c3c3769",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:32.498313Z",
     "iopub.status.busy": "2024-08-20T10:44:32.497810Z",
     "iopub.status.idle": "2024-08-20T10:44:32.506133Z",
     "shell.execute_reply": "2024-08-20T10:44:32.505286Z"
    },
    "papermill": {
     "duration": 0.031152,
     "end_time": "2024-08-20T10:44:32.508119",
     "exception": false,
     "start_time": "2024-08-20T10:44:32.476967",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class weights are [1.02225755 0.97869102] for [0 1]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "# Compute the class weights for three classes\n",
    "class_weights = compute_class_weight(class_weight=\"balanced\",\n",
    "                                     classes=np.unique(train_labels),\n",
    "                                     y=train_labels)\n",
    "print(\"Class weights are {} for {}\".format(class_weights, np.unique(train_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "51a47e2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:32.542611Z",
     "iopub.status.busy": "2024-08-20T10:44:32.542293Z",
     "iopub.status.idle": "2024-08-20T10:44:32.551290Z",
     "shell.execute_reply": "2024-08-20T10:44:32.550254Z"
    },
    "papermill": {
     "duration": 0.02871,
     "end_time": "2024-08-20T10:44:32.553514",
     "exception": false,
     "start_time": "2024-08-20T10:44:32.524804",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_23/883685092.py:1: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  pd.value_counts(train_labels)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "class\n",
       "1    1314\n",
       "0    1258\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "56dd08b0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:32.586863Z",
     "iopub.status.busy": "2024-08-20T10:44:32.586508Z",
     "iopub.status.idle": "2024-08-20T10:44:32.593406Z",
     "shell.execute_reply": "2024-08-20T10:44:32.592571Z"
    },
    "papermill": {
     "duration": 0.025342,
     "end_time": "2024-08-20T10:44:32.595296",
     "exception": false,
     "start_time": "2024-08-20T10:44:32.569954",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "weights = torch.tensor(class_weights, dtype=torch.float)\n",
    "\n",
    "weights = weights.to(device)\n",
    "cross_entropy = nn.CrossEntropyLoss()\n",
    "\n",
    "epochs = 50 ##increase the no of epochs for better accuracy \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "2f11d903",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:32.629178Z",
     "iopub.status.busy": "2024-08-20T10:44:32.628856Z",
     "iopub.status.idle": "2024-08-20T10:44:32.643844Z",
     "shell.execute_reply": "2024-08-20T10:44:32.643001Z"
    },
    "papermill": {
     "duration": 0.033972,
     "end_time": "2024-08-20T10:44:32.645634",
     "exception": false,
     "start_time": "2024-08-20T10:44:32.611662",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train():\n",
    "    model.train()\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    total_preds, total_labels = [], []\n",
    "\n",
    "    for step, batch in enumerate(train_dataloader, 0):\n",
    "        if step % 50 == 0 and not step == 0:\n",
    "            print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))\n",
    "\n",
    "#         sent_id = batch[:,0].to(device, dtype = torch.long)\n",
    "#         mask = data[:,1].to(device, dtype = torch.long)\n",
    "#         token_type_ids = data[:,2].to(device, dtype = torch.long)\n",
    "#         labels = data[:,3].to(device, dtype = torch.float)\n",
    "        \n",
    "#         token_type_ids = batch['token_type_ids'].to(device)\n",
    "        batch = [r.to(device) for r in batch]\n",
    "        sent_id, mask, token_type_ids, labels = batch\n",
    "\n",
    "        model.zero_grad()\n",
    "        preds = model(sent_id, mask, token_type_ids)\n",
    "        loss = cross_entropy(preds, labels)\n",
    "        total_loss = total_loss + loss.item()\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        optimizer.step()\n",
    "\n",
    "        preds = preds.detach().cpu().numpy()\n",
    "        total_preds.append(preds)\n",
    "        total_labels.append(labels.detach().cpu().numpy())\n",
    "\n",
    "    avg_loss = total_loss / len(train_dataloader)\n",
    "    total_preds = np.concatenate(total_preds, axis=0)\n",
    "    total_labels = np.concatenate(total_labels, axis=0)\n",
    "    return avg_loss, total_preds, total_labels\n",
    "\n",
    "def evaluate():\n",
    "    print(\"\\nEvaluating after the test: \")\n",
    "    model.eval()\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    total_preds, total_labels = [], []\n",
    "\n",
    "    for step, batch in enumerate(val_dataloader):\n",
    "        if step % 50 == 0 and not step == 0:\n",
    "            print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(val_dataloader)))\n",
    "            \n",
    "#         sent_id = batch[:,0].to(device, dtype = torch.long)\n",
    "#         mask = data[:,1].to(device, dtype = torch.long)\n",
    "#         token_type_ids = data[:,2].to(device, dtype = torch.long)\n",
    "#         labels = data[:,3].to(device, dtype = torch.float)\n",
    "\n",
    "#         token_type_ids = batch['token_type_ids'].to(device)\n",
    "        batch = [t.to(device) for t in batch]\n",
    "        sent_id, mask, token_type_ids, labels = batch\n",
    "\n",
    "        with torch.no_grad():\n",
    "            preds = model(sent_id, mask, token_type_ids)\n",
    "            loss = cross_entropy(preds, labels)\n",
    "            total_loss = total_loss + loss.item()\n",
    "            preds = preds.detach().cpu().numpy()\n",
    "            total_preds.append(preds)\n",
    "            total_labels.append(labels.detach().cpu().numpy())\n",
    "\n",
    "    avg_loss = total_loss / len(val_dataloader)\n",
    "    total_preds = np.concatenate(total_preds, axis=0)\n",
    "    total_labels = np.concatenate(total_labels, axis=0)\n",
    "    return avg_loss, total_preds, total_labels\n",
    "def calculate_accuracy(preds, labels):\n",
    "\n",
    "    preds_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    accuracy = np.sum(preds_flat == labels_flat) / len(labels_flat)\n",
    "    return accuracy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0dde143f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:44:32.679566Z",
     "iopub.status.busy": "2024-08-20T10:44:32.679235Z",
     "iopub.status.idle": "2024-08-20T10:50:58.971621Z",
     "shell.execute_reply": "2024-08-20T10:50:58.970155Z"
    },
    "papermill": {
     "duration": 386.311871,
     "end_time": "2024-08-20T10:50:58.973723",
     "exception": false,
     "start_time": "2024-08-20T10:44:32.661852",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Epoch 1 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.6925678791069403\n",
      "Validation Loss: 0.6175085306167603\n",
      "Training Accuracy: 57.08%\n",
      "Validation Accuracy: 65.11%\n",
      "\n",
      " Epoch 2 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.6258057951927185\n",
      "Validation Loss: 0.6354355911413828\n",
      "Training Accuracy: 64.74%\n",
      "Validation Accuracy: 53.89%\n",
      "\n",
      " Epoch 3 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.6156047182839092\n",
      "Validation Loss: 0.5383544017871221\n",
      "Training Accuracy: 64.89%\n",
      "Validation Accuracy: 71.03%\n",
      "\n",
      " Epoch 4 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5927572475700844\n",
      "Validation Loss: 0.5718347877264023\n",
      "Training Accuracy: 67.92%\n",
      "Validation Accuracy: 65.11%\n",
      "\n",
      " Epoch 5 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5850660117661081\n",
      "Validation Loss: 0.5047635808587074\n",
      "Training Accuracy: 68.51%\n",
      "Validation Accuracy: 73.21%\n",
      "\n",
      " Epoch 6 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5771245963689757\n",
      "Validation Loss: 0.6112248500188192\n",
      "Training Accuracy: 68.93%\n",
      "Validation Accuracy: 62.93%\n",
      "\n",
      " Epoch 7 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5685626210235968\n",
      "Validation Loss: 0.5443202952543894\n",
      "Training Accuracy: 69.98%\n",
      "Validation Accuracy: 66.98%\n",
      "\n",
      " Epoch 8 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5773887815998822\n",
      "Validation Loss: 0.811702181895574\n",
      "Training Accuracy: 71.00%\n",
      "Validation Accuracy: 54.21%\n",
      "\n",
      " Epoch 9 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.6007454446176204\n",
      "Validation Loss: 0.5005973490575949\n",
      "Training Accuracy: 68.62%\n",
      "Validation Accuracy: 71.03%\n",
      "\n",
      " Epoch 10 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5377287428553511\n",
      "Validation Loss: 0.46991171812017757\n",
      "Training Accuracy: 72.28%\n",
      "Validation Accuracy: 77.88%\n",
      "\n",
      " Epoch 11 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5287902086246304\n",
      "Validation Loss: 0.5851129392782847\n",
      "Training Accuracy: 73.60%\n",
      "Validation Accuracy: 66.36%\n",
      "\n",
      " Epoch 12 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5465484763064036\n",
      "Validation Loss: 0.632880707581838\n",
      "Training Accuracy: 73.02%\n",
      "Validation Accuracy: 60.44%\n",
      "\n",
      " Epoch 13 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.534432507142788\n",
      "Validation Loss: 0.47270021090904873\n",
      "Training Accuracy: 72.43%\n",
      "Validation Accuracy: 78.82%\n",
      "\n",
      " Epoch 14 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5134479214505452\n",
      "Validation Loss: 0.473947378496329\n",
      "Training Accuracy: 75.74%\n",
      "Validation Accuracy: 75.39%\n",
      "\n",
      " Epoch 15 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5074293300872896\n",
      "Validation Loss: 0.4647924527525902\n",
      "Training Accuracy: 76.32%\n",
      "Validation Accuracy: 76.95%\n",
      "\n",
      " Epoch 16 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.505539975515226\n",
      "Validation Loss: 0.4518366406361262\n",
      "Training Accuracy: 75.47%\n",
      "Validation Accuracy: 77.88%\n",
      "\n",
      " Epoch 17 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5143365787296761\n",
      "Validation Loss: 0.45813509821891785\n",
      "Training Accuracy: 74.69%\n",
      "Validation Accuracy: 76.95%\n",
      "\n",
      " Epoch 18 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5210385322570801\n",
      "Validation Loss: 0.45549023151397705\n",
      "Training Accuracy: 74.65%\n",
      "Validation Accuracy: 78.19%\n",
      "\n",
      " Epoch 19 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.512260460272068\n",
      "Validation Loss: 0.5791282455126444\n",
      "Training Accuracy: 74.53%\n",
      "Validation Accuracy: 66.36%\n",
      "\n",
      " Epoch 20 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.512492727942583\n",
      "Validation Loss: 0.485247477889061\n",
      "Training Accuracy: 73.95%\n",
      "Validation Accuracy: 74.45%\n",
      "\n",
      " Epoch 21 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4951843270441381\n",
      "Validation Loss: 0.4451421971122424\n",
      "Training Accuracy: 76.01%\n",
      "Validation Accuracy: 78.19%\n",
      "\n",
      " Epoch 22 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5210876348542004\n",
      "Validation Loss: 0.4784970059990883\n",
      "Training Accuracy: 74.77%\n",
      "Validation Accuracy: 75.70%\n",
      "\n",
      " Epoch 23 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.521617352235608\n",
      "Validation Loss: 0.5948563615481058\n",
      "Training Accuracy: 74.11%\n",
      "Validation Accuracy: 66.04%\n",
      "\n",
      " Epoch 24 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5226606901099042\n",
      "Validation Loss: 0.45343515276908875\n",
      "Training Accuracy: 74.07%\n",
      "Validation Accuracy: 78.19%\n",
      "\n",
      " Epoch 25 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5055509907443348\n",
      "Validation Loss: 0.47803741196791333\n",
      "Training Accuracy: 75.19%\n",
      "Validation Accuracy: 77.57%\n",
      "\n",
      " Epoch 26 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5050267090157765\n",
      "Validation Loss: 0.43975767989953357\n",
      "Training Accuracy: 75.47%\n",
      "Validation Accuracy: 78.82%\n",
      "\n",
      " Epoch 27 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.48065391982473976\n",
      "Validation Loss: 0.4519197978079319\n",
      "Training Accuracy: 76.98%\n",
      "Validation Accuracy: 77.57%\n",
      "\n",
      " Epoch 28 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5089567085591759\n",
      "Validation Loss: 0.4666433061162631\n",
      "Training Accuracy: 75.19%\n",
      "Validation Accuracy: 75.08%\n",
      "\n",
      " Epoch 29 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.494465980587936\n",
      "Validation Loss: 0.4768679738044739\n",
      "Training Accuracy: 75.74%\n",
      "Validation Accuracy: 76.32%\n",
      "\n",
      " Epoch 30 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5174569131397619\n",
      "Validation Loss: 0.44658895830313367\n",
      "Training Accuracy: 74.30%\n",
      "Validation Accuracy: 79.13%\n",
      "\n",
      " Epoch 31 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4829174025756557\n",
      "Validation Loss: 0.5370551577458779\n",
      "Training Accuracy: 78.03%\n",
      "Validation Accuracy: 69.16%\n",
      "\n",
      " Epoch 32 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.48911265847159596\n",
      "Validation Loss: 0.5499145885308584\n",
      "Training Accuracy: 76.44%\n",
      "Validation Accuracy: 69.47%\n",
      "\n",
      " Epoch 33 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4759792678239869\n",
      "Validation Loss: 0.5470955967903137\n",
      "Training Accuracy: 76.56%\n",
      "Validation Accuracy: 70.09%\n",
      "\n",
      " Epoch 34 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5048292628148707\n",
      "Validation Loss: 0.44559120138486225\n",
      "Training Accuracy: 74.81%\n",
      "Validation Accuracy: 80.06%\n",
      "\n",
      " Epoch 35 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.48981216477184764\n",
      "Validation Loss: 0.4585338632265727\n",
      "Training Accuracy: 76.36%\n",
      "Validation Accuracy: 75.70%\n",
      "\n",
      " Epoch 36 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.489656619909333\n",
      "Validation Loss: 0.511269822716713\n",
      "Training Accuracy: 76.67%\n",
      "Validation Accuracy: 71.34%\n",
      "\n",
      " Epoch 37 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.48278426905957667\n",
      "Validation Loss: 0.45401422679424286\n",
      "Training Accuracy: 77.64%\n",
      "Validation Accuracy: 77.26%\n",
      "\n",
      " Epoch 38 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4829876364731207\n",
      "Validation Loss: 0.4404156481226285\n",
      "Training Accuracy: 76.59%\n",
      "Validation Accuracy: 77.88%\n",
      "\n",
      " Epoch 39 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4800753244539587\n",
      "Validation Loss: 0.4347778186202049\n",
      "Training Accuracy: 77.49%\n",
      "Validation Accuracy: 79.44%\n",
      "\n",
      " Epoch 40 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5416067140858348\n",
      "Validation Loss: 0.42827260742584866\n",
      "Training Accuracy: 72.28%\n",
      "Validation Accuracy: 80.06%\n",
      "\n",
      " Epoch 41 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4771345334082115\n",
      "Validation Loss: 0.43815818677345914\n",
      "Training Accuracy: 76.21%\n",
      "Validation Accuracy: 79.75%\n",
      "\n",
      " Epoch 42 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.49382642493015383\n",
      "Validation Loss: 0.43377137184143066\n",
      "Training Accuracy: 76.63%\n",
      "Validation Accuracy: 79.75%\n",
      "\n",
      " Epoch 43 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.483584108876019\n",
      "Validation Loss: 0.4442533999681473\n",
      "Training Accuracy: 76.79%\n",
      "Validation Accuracy: 79.13%\n",
      "\n",
      " Epoch 44 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5122363843568941\n",
      "Validation Loss: 0.45989250764250755\n",
      "Training Accuracy: 75.70%\n",
      "Validation Accuracy: 77.57%\n",
      "\n",
      " Epoch 45 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4891212415404436\n",
      "Validation Loss: 0.4365263693034649\n",
      "Training Accuracy: 76.21%\n",
      "Validation Accuracy: 79.75%\n",
      "\n",
      " Epoch 46 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4571554002965369\n",
      "Validation Loss: 0.4350595896442731\n",
      "Training Accuracy: 78.85%\n",
      "Validation Accuracy: 79.75%\n",
      "\n",
      " Epoch 47 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4723834911497628\n",
      "Validation Loss: 0.44939280425508815\n",
      "Training Accuracy: 77.22%\n",
      "Validation Accuracy: 78.50%\n",
      "\n",
      " Epoch 48 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4920402685316598\n",
      "Validation Loss: 0.6220840116341909\n",
      "Training Accuracy: 76.87%\n",
      "Validation Accuracy: 65.11%\n",
      "\n",
      " Epoch 49 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4869711191915884\n",
      "Validation Loss: 0.4229571521282196\n",
      "Training Accuracy: 75.23%\n",
      "Validation Accuracy: 80.69%\n",
      "\n",
      " Epoch 50 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4557218257246948\n",
      "Validation Loss: 0.45387349277734756\n",
      "Training Accuracy: 78.38%\n",
      "Validation Accuracy: 77.88%\n"
     ]
    }
   ],
   "source": [
    "best_valid_loss = float('inf')\n",
    "train_losses = []\n",
    "valid_losses = []\n",
    "train_accuracies = []\n",
    "valid_accuracies = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    print('\\n Epoch {:} / {:}'.format(epoch + 1, epochs))\n",
    "    \n",
    "    train_loss, train_preds, train_labels = train()\n",
    "    valid_loss, valid_preds, valid_labels = evaluate()\n",
    "    train_accuracy = calculate_accuracy(train_preds, train_labels)\n",
    "    valid_accuracy = calculate_accuracy(valid_preds, valid_labels)\n",
    "\n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), 'saved_weights.pt')\n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    valid_losses.append(valid_loss)\n",
    "    train_accuracies.append(train_accuracy)\n",
    "    valid_accuracies.append(valid_accuracy)\n",
    "    \n",
    "    print('\\nTraining Loss: {}'.format(train_loss))\n",
    "    print('Validation Loss: {}'.format(valid_loss))\n",
    "    print('Training Accuracy: {:.2f}%'.format(train_accuracy * 100))\n",
    "    print('Validation Accuracy: {:.2f}%'.format(valid_accuracy * 100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "048a03d2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:50:59.024872Z",
     "iopub.status.busy": "2024-08-20T10:50:59.024111Z",
     "iopub.status.idle": "2024-08-20T10:50:59.849144Z",
     "shell.execute_reply": "2024-08-20T10:50:59.848002Z"
    },
    "papermill": {
     "duration": 0.85275,
     "end_time": "2024-08-20T10:50:59.851671",
     "exception": false,
     "start_time": "2024-08-20T10:50:58.998921",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.66      0.71       157\n",
      "           1       0.72      0.82      0.77       165\n",
      "\n",
      "    accuracy                           0.74       322\n",
      "   macro avg       0.75      0.74      0.74       322\n",
      "weighted avg       0.75      0.74      0.74       322\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "with torch.no_grad():\n",
    "  preds = model(test_seq.to(device), test_mask.to(device), test_token_type_ids.to(device))\n",
    "  preds = preds.detach().cpu().numpy()\n",
    "pred = np.argmax(preds, axis = 1)\n",
    "print(classification_report(test_y, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b83a8a2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-20T10:50:59.903043Z",
     "iopub.status.busy": "2024-08-20T10:50:59.902721Z",
     "iopub.status.idle": "2024-08-20T10:50:59.932247Z",
     "shell.execute_reply": "2024-08-20T10:50:59.931262Z"
    },
    "papermill": {
     "duration": 0.056724,
     "end_time": "2024-08-20T10:50:59.934340",
     "exception": false,
     "start_time": "2024-08-20T10:50:59.877616",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted sentiment: Positive\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2760: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "def predict_user_input(input_text, model, tokenizer, device):\n",
    "    inputs = tokenizer.encode_plus(\n",
    "        input_text,\n",
    "        add_special_tokens=True,\n",
    "        max_length=max_seq_len,\n",
    "        pad_to_max_length=True,\n",
    "        return_attention_mask=True,\n",
    "        return_token_type_ids=True,\n",
    "        return_tensors='pt',\n",
    "    )\n",
    "    input_ids = inputs['input_ids'].to(device)\n",
    "    attention_mask = inputs['attention_mask'].to(device)\n",
    "    token_type_ids = inputs['token_type_ids'].to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        outputs = model(input_ids, attention_mask, token_type_ids)\n",
    "        preds = torch.argmax(outputs, dim=1)\n",
    "    \n",
    "    return preds.item()\n",
    "max_seq_len=50\n",
    "input_text = \"what a badass character Arthur is, rckstr is the bst\"\n",
    "predicted_class = predict_user_input(input_text, model, tokenizer, device)\n",
    "if(predicted_class==0):\n",
    "    print('Predicted sentiment: Negative')\n",
    "else:\n",
    "    print('Predicted sentiment: Positive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68fb3499",
   "metadata": {
    "papermill": {
     "duration": 0.024314,
     "end_time": "2024-08-20T10:50:59.983929",
     "exception": false,
     "start_time": "2024-08-20T10:50:59.959615",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "nvidiaTeslaT4",
   "dataSources": [
    {
     "datasetId": 5557640,
     "sourceId": 9193227,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30747,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 406.593095,
   "end_time": "2024-08-20T10:51:01.532737",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-08-20T10:44:14.939642",
   "version": "2.5.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "023437ced18043aebb9710f5833563ed": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "0462ec2b832f4028aa6f63e7c676ac9e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "0555388522be4b1aada7bd5d3a211e5b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_6e599058c3a7487aace61cf376488f71",
       "max": 48.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_851dbbaebe6d4e04ba1477853d069f20",
       "value": 48.0
      }
     },
     "0acb0cb27e754db8a231e54d18fbaa82": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "18b19db434964a3eaa99eea69952f92d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_023437ced18043aebb9710f5833563ed",
       "max": 570.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_1e1066da3599418597330dfd8f49bfcd",
       "value": 570.0
      }
     },
     "1e1066da3599418597330dfd8f49bfcd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "1f007a6c4fb046d7a81f5873de8336d5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d7dfe2c4ff1e494a880d99d8eb9d6f4f",
       "placeholder": "​",
       "style": "IPY_MODEL_40249364f2ef41c3b8ce8571a8413744",
       "value": "config.json: 100%"
      }
     },
     "24277973d47248bc9ad8618a060055cf": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "242fd53952bf467f9406b327f56ed850": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_0acb0cb27e754db8a231e54d18fbaa82",
       "max": 231508.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_634e446be4aa4d17b58baa4c92f13fec",
       "value": 231508.0
      }
     },
     "29b5aa14324b4588af215fd483e11da6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2b9a61d3da014f429b4bbb483d2a0821": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "357bb26a0e3544099d8115ea269da68a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "3df6f83348cd4af0be9de2622256eba4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "40249364f2ef41c3b8ce8571a8413744": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4804a103c1844ca5a1a2326ed1390334": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_29b5aa14324b4588af215fd483e11da6",
       "placeholder": "​",
       "style": "IPY_MODEL_d9d074b918b042a09b2fff31e06f4f6d",
       "value": " 570/570 [00:00&lt;00:00, 50.2kB/s]"
      }
     },
     "4e51bdc8bcf54b09ba036d97e7a52182": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5989aafd93204900968ac259fd370b98": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "61da6385f06e4e7f9a65c5c4555b1343": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f81d6fb81b094c059d36474964149e27",
       "placeholder": "​",
       "style": "IPY_MODEL_8152c10be29d43d89111b8238351351c",
       "value": "model.safetensors: 100%"
      }
     },
     "634e446be4aa4d17b58baa4c92f13fec": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "651a2eea54e74e8a88a7285854196003": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_dbae603545aa4e6fb6a2ff5b28fdff70",
        "IPY_MODEL_242fd53952bf467f9406b327f56ed850",
        "IPY_MODEL_777cc3049a3d41aaa5dab1977b0c454d"
       ],
       "layout": "IPY_MODEL_2b9a61d3da014f429b4bbb483d2a0821"
      }
     },
     "66c431a9a2a54a86b748ab8b3482d66d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_0462ec2b832f4028aa6f63e7c676ac9e",
       "placeholder": "​",
       "style": "IPY_MODEL_5989aafd93204900968ac259fd370b98",
       "value": "tokenizer_config.json: 100%"
      }
     },
     "67e76ac7f0f74b3d9d92216bd4faf95f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a9e716c062de4bffa7cc3fc6fd653349",
       "placeholder": "​",
       "style": "IPY_MODEL_fe145aaa38864d41b4bb763efdd3f8a3",
       "value": " 466k/466k [00:00&lt;00:00, 6.58MB/s]"
      }
     },
     "6e599058c3a7487aace61cf376488f71": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "70896c6db7a64b819e5d9b43cefb83b1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7247b6cd205c4e52a83fbc246d894b94": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_1f007a6c4fb046d7a81f5873de8336d5",
        "IPY_MODEL_18b19db434964a3eaa99eea69952f92d",
        "IPY_MODEL_4804a103c1844ca5a1a2326ed1390334"
       ],
       "layout": "IPY_MODEL_868d921249754851aa93bf5b652034d2"
      }
     },
     "777cc3049a3d41aaa5dab1977b0c454d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c39fa19551d046218a52703bc9dbbd58",
       "placeholder": "​",
       "style": "IPY_MODEL_e9535eb11764442d86ba0a345cee7f20",
       "value": " 232k/232k [00:00&lt;00:00, 4.36MB/s]"
      }
     },
     "7cd1f7b1954a4bc3987b0498056d68d6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c939bf268bae43ae96e78f080b8c9419",
       "max": 466062.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_f4f38a33096948448d023a61a0793bb6",
       "value": 466062.0
      }
     },
     "8152c10be29d43d89111b8238351351c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "851dbbaebe6d4e04ba1477853d069f20": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "868d921249754851aa93bf5b652034d2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "894f31b688e84802bcc827f3f56b6ff0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9271579758944ba48cee5ffb45b83992": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_894f31b688e84802bcc827f3f56b6ff0",
       "placeholder": "​",
       "style": "IPY_MODEL_357bb26a0e3544099d8115ea269da68a",
       "value": "tokenizer.json: 100%"
      }
     },
     "a774f698e7bc44d2b1c55cf88c93d036": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "a9e716c062de4bffa7cc3fc6fd653349": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b1816a2351a24413962004204ef0661d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "c39fa19551d046218a52703bc9dbbd58": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c939bf268bae43ae96e78f080b8c9419": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cafb4ba895af488ab58a18021c6b902b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "d1180473bf2d447099ccc9000e4732ea": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_70896c6db7a64b819e5d9b43cefb83b1",
       "max": 440449768.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_e70e7e1363b94ec998a4b7438727fde4",
       "value": 440449768.0
      }
     },
     "d7dfe2c4ff1e494a880d99d8eb9d6f4f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d7e6419096a147d2bd20cfd900857958": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d9d074b918b042a09b2fff31e06f4f6d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "dbae603545aa4e6fb6a2ff5b28fdff70": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_4e51bdc8bcf54b09ba036d97e7a52182",
       "placeholder": "​",
       "style": "IPY_MODEL_a774f698e7bc44d2b1c55cf88c93d036",
       "value": "vocab.txt: 100%"
      }
     },
     "e26b1e1beb8d449c8be2b8476b3ac1ee": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_9271579758944ba48cee5ffb45b83992",
        "IPY_MODEL_7cd1f7b1954a4bc3987b0498056d68d6",
        "IPY_MODEL_67e76ac7f0f74b3d9d92216bd4faf95f"
       ],
       "layout": "IPY_MODEL_f80571d13fdb4f45a07d4f15fae8d43f"
      }
     },
     "e517e4c7ca1a4833819de97a98edd511": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d7e6419096a147d2bd20cfd900857958",
       "placeholder": "​",
       "style": "IPY_MODEL_b1816a2351a24413962004204ef0661d",
       "value": " 48.0/48.0 [00:00&lt;00:00, 4.14kB/s]"
      }
     },
     "e70e7e1363b94ec998a4b7438727fde4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "e9535eb11764442d86ba0a345cee7f20": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f2e5f47e4bb84c4c883d6fddb59356e7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_66c431a9a2a54a86b748ab8b3482d66d",
        "IPY_MODEL_0555388522be4b1aada7bd5d3a211e5b",
        "IPY_MODEL_e517e4c7ca1a4833819de97a98edd511"
       ],
       "layout": "IPY_MODEL_24277973d47248bc9ad8618a060055cf"
      }
     },
     "f3315154c31b488a9a5c0da5b0abf3b9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f4f38a33096948448d023a61a0793bb6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "f80571d13fdb4f45a07d4f15fae8d43f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f81d6fb81b094c059d36474964149e27": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fd8088a298464de78ed730d6fa518b56": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_61da6385f06e4e7f9a65c5c4555b1343",
        "IPY_MODEL_d1180473bf2d447099ccc9000e4732ea",
        "IPY_MODEL_ffa85cb229b048f680a21024be7e68c4"
       ],
       "layout": "IPY_MODEL_3df6f83348cd4af0be9de2622256eba4"
      }
     },
     "fe145aaa38864d41b4bb763efdd3f8a3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ffa85cb229b048f680a21024be7e68c4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f3315154c31b488a9a5c0da5b0abf3b9",
       "placeholder": "​",
       "style": "IPY_MODEL_cafb4ba895af488ab58a18021c6b902b",
       "value": " 440M/440M [00:02&lt;00:00, 210MB/s]"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}