Fine_Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "60a6113e-3930-447e-a207-dab8aa970192",
   "metadata": {},
   "source": [
    "# Fine-tune own financial sentiment NLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ba1890d4-00a5-4f6f-ae00-db6583ac50ef",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "from transformers import BertTokenizer, Trainer, BertForSequenceClassification, TrainingArguments\n",
    "from datasets import Dataset\n",
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a262ae6c-f7b5-42c7-8608-c538e101b60c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('2.3.0+cu121', '4.41.1')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tested in transformers==4.18.0, pytorch==1.7.1 \n",
    "import torch\n",
    "import transformers\n",
    "torch.__version__, transformers.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "876136d0-3f3d-4746-8b27-24cebc05cb6a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4a57996-ec94-44dc-aa04-750676f090cc",
   "metadata": {},
   "source": [
    "# Data loading\n",
    "\n",
    " '1' denotes a positive sentiment, '2' signifies a negative sentiment, and '0' indicates a neutral sentiment. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "bdc5c97a-b738-448c-8ef2-9471d0f54089",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Teollisuuden Voima Oyj , the Finnish utility k...</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sanofi poaches AstraZeneca scientist as new re...</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Starbucks says the workers violated safety pol...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>$brcm raises revenue forecast</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Google parent Alphabet Inc. reported revenue a...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76767</th>\n",
       "      <td>BP, Statoil, to Withdraw Staff From Algeria Fo...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76768</th>\n",
       "      <td>NEW YORK — A fire broke out late Wednesday at ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76769</th>\n",
       "      <td>Operating profit margin increased from 11.2 % ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76770</th>\n",
       "      <td>$vxx adding to position here !</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76771</th>\n",
       "      <td>Leading chipmaker Nvidia could be heading for ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>76772 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text     label\n",
       "0      Teollisuuden Voima Oyj , the Finnish utility k...   neutral\n",
       "1      Sanofi poaches AstraZeneca scientist as new re...   neutral\n",
       "2      Starbucks says the workers violated safety pol...  negative\n",
       "3                          $brcm raises revenue forecast  positive\n",
       "4      Google parent Alphabet Inc. reported revenue a...  negative\n",
       "...                                                  ...       ...\n",
       "76767  BP, Statoil, to Withdraw Staff From Algeria Fo...  negative\n",
       "76768  NEW YORK — A fire broke out late Wednesday at ...  negative\n",
       "76769  Operating profit margin increased from 11.2 % ...  positive\n",
       "76770                     $vxx adding to position here !  positive\n",
       "76771  Leading chipmaker Nvidia could be heading for ...  positive\n",
       "\n",
       "[76772 rows x 2 columns]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = pd.read_parquet('fingpt_sentiment_train-00000-of-00001-7790814d50128a07.parquet')\n",
    "\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "f51d166c-dd15-4e6f-a2db-c55ca82ea2f5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Teollisuuden Voima Oyj , the Finnish utility k...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sanofi poaches AstraZeneca scientist as new re...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Starbucks says the workers violated safety pol...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>$brcm raises revenue forecast</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Google parent Alphabet Inc. reported revenue a...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76767</th>\n",
       "      <td>BP, Statoil, to Withdraw Staff From Algeria Fo...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76768</th>\n",
       "      <td>NEW YORK — A fire broke out late Wednesday at ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76769</th>\n",
       "      <td>Operating profit margin increased from 11.2 % ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76770</th>\n",
       "      <td>$vxx adding to position here !</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76771</th>\n",
       "      <td>Leading chipmaker Nvidia could be heading for ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>76772 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  label\n",
       "0      Teollisuuden Voima Oyj , the Finnish utility k...      0\n",
       "1      Sanofi poaches AstraZeneca scientist as new re...      0\n",
       "2      Starbucks says the workers violated safety pol...      2\n",
       "3                          $brcm raises revenue forecast      1\n",
       "4      Google parent Alphabet Inc. reported revenue a...      2\n",
       "...                                                  ...    ...\n",
       "76767  BP, Statoil, to Withdraw Staff From Algeria Fo...      2\n",
       "76768  NEW YORK — A fire broke out late Wednesday at ...      2\n",
       "76769  Operating profit margin increased from 11.2 % ...      1\n",
       "76770                     $vxx adding to position here !      1\n",
       "76771  Leading chipmaker Nvidia could be heading for ...      1\n",
       "\n",
       "[76772 rows x 2 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_mapping = {'neutral': 0, 'positive': 1, 'negative': 2}\n",
    "\n",
    "# Apply the mapping to the 'label' column\n",
    "df1['label'] = df1['label'].map(label_mapping)\n",
    "\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "21712d32-311c-41c2-a8d6-138959d1f6b5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>According to Gran , the company has no plans t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Technopolis plans to develop in stages an area...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>With the new production plant the company woul...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>According to the company 's updated strategy f...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>For the last quarter of 2010 , Componenta 's n...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4212</th>\n",
       "      <td>HELSINKI Thomson Financial - Shares in Cargote...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4213</th>\n",
       "      <td>LONDON MarketWatch -- Share prices ended lower...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4214</th>\n",
       "      <td>Rinkuskiai 's beer sales fell by 6.5 per cent ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4215</th>\n",
       "      <td>Operating profit fell to EUR 35.4 mn from EUR ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4216</th>\n",
       "      <td>Sales in Finland decreased by 10.5 % in Januar...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4217 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  label\n",
       "0     According to Gran , the company has no plans t...      0\n",
       "1     Technopolis plans to develop in stages an area...      0\n",
       "2     With the new production plant the company woul...      1\n",
       "3     According to the company 's updated strategy f...      1\n",
       "4     For the last quarter of 2010 , Componenta 's n...      1\n",
       "...                                                 ...    ...\n",
       "4212  HELSINKI Thomson Financial - Shares in Cargote...      2\n",
       "4213  LONDON MarketWatch -- Share prices ended lower...      2\n",
       "4214  Rinkuskiai 's beer sales fell by 6.5 per cent ...      0\n",
       "4215  Operating profit fell to EUR 35.4 mn from EUR ...      2\n",
       "4216  Sales in Finland decreased by 10.5 % in Januar...      2\n",
       "\n",
       "[4217 rows x 2 columns]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = pd.read_parquet('financial_phrasebank-00000-of-00001-46d5c1a7817abe3d.parquet')\n",
    "\n",
    "# Apply the mapping to the 'label' column\n",
    "df2['label'] = df2['label'].map(label_mapping)\n",
    "\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "39f302be-afe8-4c0e-85eb-8559496e70a5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Altia 's operating profit jumped to EUR 47 mil...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The agreement was signed with Biohit Healthcar...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kesko pursues a strategy of healthy , focused ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Vaisala , headquartered in Helsinki in Finland...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Also , a six-year historic analysis is provide...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3872</th>\n",
       "      <td>Dubai Nokia has announced the launch of `` Com...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3873</th>\n",
       "      <td>MADISON , Wis. , Feb. 6 - PRNewswire - -- Fisk...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3874</th>\n",
       "      <td>The report provides a comprehensive insight in...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3875</th>\n",
       "      <td>Pharmaceuticals - Netherlands This brand-new m...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3876</th>\n",
       "      <td>The technology will become available to busine...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3877 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  label\n",
       "0     Altia 's operating profit jumped to EUR 47 mil...      1\n",
       "1     The agreement was signed with Biohit Healthcar...      1\n",
       "2     Kesko pursues a strategy of healthy , focused ...      1\n",
       "3     Vaisala , headquartered in Helsinki in Finland...      0\n",
       "4     Also , a six-year historic analysis is provide...      0\n",
       "...                                                 ...    ...\n",
       "3872  Dubai Nokia has announced the launch of `` Com...      1\n",
       "3873  MADISON , Wis. , Feb. 6 - PRNewswire - -- Fisk...      1\n",
       "3874  The report provides a comprehensive insight in...      1\n",
       "3875  Pharmaceuticals - Netherlands This brand-new m...      0\n",
       "3876  The technology will become available to busine...      0\n",
       "\n",
       "[3877 rows x 2 columns]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3= pd.read_parquet('auditor_sentiment-00000-of-00001-7c7834e29b677695.parquet')\n",
    "\n",
    "# Apply the mapping to the 'label' column\n",
    "df3['label'] = df3['label'].map(label_mapping)\n",
    "\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "395610ac-50bd-42a8-b7b2-4014a5ba5152",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>$BYND - JPMorgan reels in expectations on Beyo...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>$CCL $RCL - Nomura points to bookings weakness...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>$CX - Cemex cut at Credit Suisse, J.P. Morgan ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>$ESS: BTIG Research cuts to Neutral https://t....</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>$FNKO - Funko slides after Piper Jaffray PT cu...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9538</th>\n",
       "      <td>The Week's Gainers and Losers on the Stoxx Eur...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9539</th>\n",
       "      <td>Tupperware Brands among consumer gainers; Unil...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9540</th>\n",
       "      <td>vTv Therapeutics leads healthcare gainers; Myo...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9541</th>\n",
       "      <td>WORK, XPO, PYX and AMKR among after hour movers</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9542</th>\n",
       "      <td>YNDX, I, QD and OESX among tech movers</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9543 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  label\n",
       "0     $BYND - JPMorgan reels in expectations on Beyo...      2\n",
       "1     $CCL $RCL - Nomura points to bookings weakness...      2\n",
       "2     $CX - Cemex cut at Credit Suisse, J.P. Morgan ...      2\n",
       "3     $ESS: BTIG Research cuts to Neutral https://t....      2\n",
       "4     $FNKO - Funko slides after Piper Jaffray PT cu...      2\n",
       "...                                                 ...    ...\n",
       "9538  The Week's Gainers and Losers on the Stoxx Eur...      0\n",
       "9539  Tupperware Brands among consumer gainers; Unil...      0\n",
       "9540  vTv Therapeutics leads healthcare gainers; Myo...      0\n",
       "9541    WORK, XPO, PYX and AMKR among after hour movers      0\n",
       "9542             YNDX, I, QD and OESX among tech movers      0\n",
       "\n",
       "[9543 rows x 2 columns]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df4 = pd.read_parquet('twitter_financial_news_sentiment-00000-of-00001-e4ff9b93e3f0bcb7.parquet')\n",
    "\n",
    "label_mapping = {'neutral': 0, 'positive': 1, 'negative': 2}\n",
    "\n",
    "# Apply the mapping to the 'label' column\n",
    "df4['label'] = df4['label'].map(label_mapping)\n",
    "\n",
    "df4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "5e8eb064-79ef-4ebe-aa1c-d12224c58bb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>According to Gran , the company has no plans t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Technopolis plans to develop in stages an area...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The international electronic industry company ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>With the new production plant the company woul...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>According to the company 's updated strategy f...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4841</th>\n",
       "      <td>LONDON MarketWatch -- Share prices ended lower...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4842</th>\n",
       "      <td>Rinkuskiai 's beer sales fell by 6.5 per cent ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4843</th>\n",
       "      <td>Operating profit fell to EUR 35.4 mn from EUR ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4844</th>\n",
       "      <td>Net sales of the Paper segment decreased to EU...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4845</th>\n",
       "      <td>Sales in Finland decreased by 10.5 % in Januar...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4846 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  label\n",
       "0     According to Gran , the company has no plans t...      0\n",
       "1     Technopolis plans to develop in stages an area...      0\n",
       "2     The international electronic industry company ...      2\n",
       "3     With the new production plant the company woul...      1\n",
       "4     According to the company 's updated strategy f...      1\n",
       "...                                                 ...    ...\n",
       "4841  LONDON MarketWatch -- Share prices ended lower...      2\n",
       "4842  Rinkuskiai 's beer sales fell by 6.5 per cent ...      0\n",
       "4843  Operating profit fell to EUR 35.4 mn from EUR ...      2\n",
       "4844  Net sales of the Paper segment decreased to EU...      2\n",
       "4845  Sales in Finland decreased by 10.5 % in Januar...      2\n",
       "\n",
       "[4846 rows x 2 columns]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Try reading the file with a different encoding\n",
    "df5 = pd.read_csv('sentiment_finance.csv', encoding='latin1')\n",
    "\n",
    "# Apply the label mapping\n",
    "label_mapping = {'neutral': 0, 'positive': 1, 'negative': 2}\n",
    "df5['label'] = df5['label'].map(label_mapping)\n",
    "df5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "0285c922-54ec-4bc2-b938-e732c66384d6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Global COVID-19 death toll exceeds 4 million.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>reports 67,208 new COVID-19 cases, 2,330 deaths.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>China reports 23 new COVID-19 cases versus 19 ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>India records 91,702 new COVID-19 cases over p...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sharply raises COVID-19 death toll prompting c...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>U.S. equity futures were trading higher the mo...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>China, US commerce chiefs to cooperate on hand...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>Stock investors celebrate red-hot five-quarter...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>Wall Streetâs roaring 2021 is as good as it ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>Investor sees 'incredibly strong earnings' in ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>211 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  label\n",
       "0       Global COVID-19 death toll exceeds 4 million.       2\n",
       "1    reports 67,208 new COVID-19 cases, 2,330 deaths.       2\n",
       "2    China reports 23 new COVID-19 cases versus 19 ...      2\n",
       "3    India records 91,702 new COVID-19 cases over p...      2\n",
       "4    sharply raises COVID-19 death toll prompting c...      2\n",
       "..                                                 ...    ...\n",
       "206  U.S. equity futures were trading higher the mo...      1\n",
       "207  China, US commerce chiefs to cooperate on hand...      1\n",
       "208  Stock investors celebrate red-hot five-quarter...      1\n",
       "209  Wall Streetâs roaring 2021 is as good as it ...      1\n",
       "210  Investor sees 'incredibly strong earnings' in ...      1\n",
       "\n",
       "[211 rows x 2 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df6 = pd.read_csv('Financial_News_Sentiment.csv', encoding='latin1')\n",
    "df6 = df6.rename(columns={'text': 'text', 'sentiment': 'label'})\n",
    "df6['label'] = df6['label'].replace({0: 2, 1: 0, 2: 1})\n",
    "df6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "2b1dcff1-29ec-4053-89b5-be5f3fe50f87",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The GeoSolutions technology will leverage Bene...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>$ESI on lows, down $1.50 to $2.50 BK a real po...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>For the last quarter of 2010 , Componenta 's n...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>According to the Finnish-Russian Chamber of Co...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Swedish buyout firm has sold its remaining...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5837</th>\n",
       "      <td>RISING costs have forced packaging producer Hu...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5838</th>\n",
       "      <td>Nordic Walking was first used as a summer trai...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5839</th>\n",
       "      <td>According shipping company Viking Line , the E...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5840</th>\n",
       "      <td>In the building and home improvement trade , s...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5841</th>\n",
       "      <td>HELSINKI AFX - KCI Konecranes said it has won ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5842 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  label\n",
       "0     The GeoSolutions technology will leverage Bene...      1\n",
       "1     $ESI on lows, down $1.50 to $2.50 BK a real po...      2\n",
       "2     For the last quarter of 2010 , Componenta 's n...      1\n",
       "3     According to the Finnish-Russian Chamber of Co...      0\n",
       "4     The Swedish buyout firm has sold its remaining...      0\n",
       "...                                                 ...    ...\n",
       "5837  RISING costs have forced packaging producer Hu...      2\n",
       "5838  Nordic Walking was first used as a summer trai...      0\n",
       "5839  According shipping company Viking Line , the E...      0\n",
       "5840  In the building and home improvement trade , s...      0\n",
       "5841  HELSINKI AFX - KCI Konecranes said it has won ...      1\n",
       "\n",
       "[5842 rows x 2 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df7 = pd.read_csv('data_2.csv')\n",
    "df7.dropna(how='all')\n",
    "df7 = df7.rename(columns={'Sentence': 'text', 'Sentiment': 'label'})\n",
    "# Apply the label mapping\n",
    "label_mapping = {'neutral': 0, 'positive': 1, 'negative': 2}\n",
    "df7['label'] = df7['label'].map(label_mapping)\n",
    "df7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "5d8b6905-64e9-462a-9375-880cd69e0da3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Still short $LNG from $11.70 area...next stop ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>$PLUG bear raid</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How Kraft-Heinz Merger Came Together in Speedy...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Slump in Weir leads FTSE down from record high</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>$AAPL bounces off support, it seems</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>956</th>\n",
       "      <td>Morrisons finance chief to fill gap as CEO lea...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>957</th>\n",
       "      <td>End Of Day Scan: Bullish MA  Crossovers $BBVA ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>958</th>\n",
       "      <td>Japan's Nikkei lands Financial Times in $1.3 b...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959</th>\n",
       "      <td>Double bottom with handle buy point of 56.49 $...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>960</th>\n",
       "      <td>$SONC Amazing run since middle of March - obvi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>961 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  label\n",
       "0    Still short $LNG from $11.70 area...next stop ...      2\n",
       "1                                      $PLUG bear raid      2\n",
       "2    How Kraft-Heinz Merger Came Together in Speedy...      1\n",
       "3       Slump in Weir leads FTSE down from record high      2\n",
       "4                  $AAPL bounces off support, it seems      1\n",
       "..                                                 ...    ...\n",
       "956  Morrisons finance chief to fill gap as CEO lea...      2\n",
       "957  End Of Day Scan: Bullish MA  Crossovers $BBVA ...      1\n",
       "958  Japan's Nikkei lands Financial Times in $1.3 b...      1\n",
       "959  Double bottom with handle buy point of 56.49 $...      1\n",
       "960  $SONC Amazing run since middle of March - obvi...      1\n",
       "\n",
       "[961 rows x 2 columns]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df8 = pd.read_parquet('fiqa_2018-00000-of-00001-ea3e4fffb9862c66.parquet')\n",
    "# Apply the label mapping\n",
    "label_mapping = {'neutral': 0, 'positive': 1, 'negative': 2}\n",
    "df8['label'] = df8['label'].map(label_mapping)\n",
    "df8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "c92c2920-d297-45df-a499-c52a8ea1ac3d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>106269.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.764475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.762507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               label\n",
       "count  106269.000000\n",
       "mean        0.764475\n",
       "std         0.762507\n",
       "min         0.000000\n",
       "25%         0.000000\n",
       "50%         1.000000\n",
       "75%         1.000000\n",
       "max         2.000000"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8], ignore_index=True)\n",
    "combined_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8910dcb1-6cbc-4878-9602-b48686db4c7b",
   "metadata": {},
   "source": [
    "# Sequential Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fc6da3f-f863-47dc-8e84-ffe68895acd9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# train df1 and df2 now\n",
    "dataframes = [df1, df2]\n",
    "\n",
    "# Initialize tokenizer and model\n",
    "tokenizer = BertTokenizer.from_pretrained('yiyanghkust/finbert-pretrain')\n",
    "model = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-pretrain', num_labels=3)\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    evaluation_strategy='epoch',\n",
    "    save_strategy='epoch',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=32,\n",
    "    num_train_epochs=1,  # Train for 1 epoch per dataframe\n",
    "    weight_decay=0.01,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model='accuracy',\n",
    ")\n",
    "\n",
    "# Training loop over individual dataframes\n",
    "for idx, df in enumerate(dataframes):\n",
    "    print(f\"Training on dataframe {idx+1}/{len(dataframes)}\")\n",
    "    \n",
    "    # Split dataframe into train, validation, and test sets\n",
    "    df_train, df_test = train_test_split(df, stratify=df['label'], test_size=0.2, random_state=42)\n",
    "    df_train, df_val = train_test_split(df_train, stratify=df_train['label'], test_size=0.2, random_state=42)\n",
    "    \n",
    "    # Convert to datasets\n",
    "    dataset_train = Dataset.from_pandas(df_train)\n",
    "    dataset_val = Dataset.from_pandas(df_val)\n",
    "    dataset_test = Dataset.from_pandas(df_test)\n",
    "    \n",
    "    # Tokenize datasets\n",
    "    dataset_train = dataset_train.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_val = dataset_val.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    \n",
    "    # Set format for PyTorch\n",
    "    dataset_train.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_val.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    \n",
    "    # Initialize Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=args,\n",
    "        train_dataset=dataset_train,\n",
    "        eval_dataset=dataset_val,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "    \n",
    "    # Train the model\n",
    "    trainer.train()\n",
    "    \n",
    "    # Save the model after each dataframe\n",
    "    model.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    tokenizer.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    \n",
    "    # Optionally, evaluate on the test set after each dataframe\n",
    "    eval_results = trainer.evaluate(eval_dataset=dataset_test)\n",
    "    print(f\"Eval results for dataframe {idx+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9586e1a2-7573-4db6-b0ba-fb7afae45ef5",
   "metadata": {},
   "source": [
    "# try using model 2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3032722c-819a-4e12-8fa7-40607bbb1250",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import BertForSequenceClassification, BertTokenizer\n",
    "\n",
    "# Load the saved model and tokenizer\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_2')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_2')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4d2ab15-8b4b-49b5-8b38-8718b496e39b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "# Sample texts for prediction\n",
    "texts = [\"The company reported a significant increase in revenue.\", \n",
    "         \"The stock price dropped after the announcement of the new policy.\"]\n",
    "\n",
    "# Tokenize the input texts\n",
    "inputs = tokenizer(texts, truncation=True, padding=True, max_length=128, return_tensors='pt')\n",
    "\n",
    "# Make predictions\n",
    "model.eval()  # Set the model to evaluation mode\n",
    "with torch.no_grad():\n",
    "    outputs = model(**inputs)\n",
    "\n",
    "# Get the predicted class labels\n",
    "predictions = torch.argmax(outputs.logits, dim=1)\n",
    "\n",
    "# Convert predictions to human-readable labels (assuming 0, 1, 2 correspond to the sentiment classes)\n",
    "label_map = {0: 'neutral', 1: 'positive', 2: 'negative'}\n",
    "predicted_labels = [label_map[pred.item()] for pred in predictions]\n",
    "\n",
    "# Print the predictions\n",
    "for text, label in zip(texts, predicted_labels):\n",
    "    print(f\"Text: {text}\\nPredicted label: {label}\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd910ba5-599f-4b6e-aca3-87b8633e9942",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_testing = pd.read_parquet('fiqa_2018-00000-of-00001-ea3e4fffb9862c66.parquet')\n",
    "label_mapping = {'neutral': 0, 'positive': 1, 'negative': 2}\n",
    "\n",
    "# Apply the mapping to the 'label' column\n",
    "df_testing['label'] = df_testing['label'].map(label_mapping)\n",
    "\n",
    "df_testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a049e138-453b-42c0-985d-5e4b1db98423",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import Trainer, TrainingArguments\n",
    "from datasets import Dataset\n",
    "import pandas as pd\n",
    "\n",
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fd1ec06a-386e-44a3-ad07-ec48eb6e60ae",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A few employees would remain at the Oulu plant...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Comparable net sales are expected to increase ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Tesla is recalling 2,700 Model X cars: https:/...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Finnish software developer Done Solutions Oyj ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Compagnie de Financement Foncier - Is to issue...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164</th>\n",
       "      <td>In 2010 , the Marimekko Group s net sales were...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1165</th>\n",
       "      <td>Nokia said it still expects to sell 150 more m...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1166</th>\n",
       "      <td>PNC, Goldman Sachs Receive Dividend Hike Green...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1167</th>\n",
       "      <td>The company , employing 6,400 , reported net s...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1168</th>\n",
       "      <td>To see more of New Haven Register , or to subs...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1169 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  label\n",
       "0     A few employees would remain at the Oulu plant...      0\n",
       "1     Comparable net sales are expected to increase ...      1\n",
       "2     Tesla is recalling 2,700 Model X cars: https:/...      2\n",
       "3     Finnish software developer Done Solutions Oyj ...      1\n",
       "4     Compagnie de Financement Foncier - Is to issue...      0\n",
       "...                                                 ...    ...\n",
       "1164  In 2010 , the Marimekko Group s net sales were...      0\n",
       "1165  Nokia said it still expects to sell 150 more m...      0\n",
       "1166  PNC, Goldman Sachs Receive Dividend Hike Green...      1\n",
       "1167  The company , employing 6,400 , reported net s...      0\n",
       "1168  To see more of New Haven Register , or to subs...      0\n",
       "\n",
       "[1169 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_testing2 = pd.read_csv('test.csv')\n",
    "df_testing2 = df_testing2.rename(columns={'Sentence': 'text', 'Sentiment': 'label'})\n",
    "# Replace the values in the 'label' column\n",
    "df_testing2['label'] = df_testing2['label'].replace({0: 2, 1: 0, 2: 1})\n",
    "df_testing2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f43ae06-e86a-460f-9f49-eb484a886fe7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing2)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feec48c7-e840-4965-bab7-20f5d01b4c58",
   "metadata": {},
   "source": [
    "# keep training df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fde70fa-2c72-471b-8902-1c68ecd8996d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataframes = [df3, df4]\n",
    "\n",
    "# Load the last saved model\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_2')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_2')\n",
    "\n",
    "# Continue training loop over the next set of dataframes\n",
    "for idx, df in enumerate(dataframes, start=2):  # Start index from 2 to continue numbering\n",
    "    print(f\"Training on dataframe {idx+1}/{len(dataframes)+2}\")\n",
    "    \n",
    "    # Split dataframe into train, validation, and test sets\n",
    "    df_train, df_test = train_test_split(df, stratify=df['label'], test_size=0.2, random_state=42)\n",
    "    df_train, df_val = train_test_split(df_train, stratify=df_train['label'], test_size=0.2, random_state=42)\n",
    "    \n",
    "    # Convert to datasets\n",
    "    dataset_train = Dataset.from_pandas(df_train)\n",
    "    dataset_val = Dataset.from_pandas(df_val)\n",
    "    dataset_test = Dataset.from_pandas(df_test)\n",
    "    \n",
    "    # Tokenize datasets\n",
    "    dataset_train = dataset_train.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_val = dataset_val.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    \n",
    "    # Set format for PyTorch\n",
    "    dataset_train.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_val.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    \n",
    "    # Initialize Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=args,\n",
    "        train_dataset=dataset_train,\n",
    "        eval_dataset=dataset_val,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "    \n",
    "    # Train the model\n",
    "    trainer.train()\n",
    "    \n",
    "    # Save the model after each dataframe\n",
    "    model.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    tokenizer.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    \n",
    "    # Optionally, evaluate on the test set after each dataframe\n",
    "    eval_results = trainer.evaluate(eval_dataset=dataset_test)\n",
    "    print(f\"Eval results for dataframe {idx+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41334399-0859-41ce-83b7-39c0d628859f",
   "metadata": {},
   "source": [
    "# try using model 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88761438-4126-4b96-8d6a-ad3ab20bab62",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load the saved model and tokenizer\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_4')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6757983-06ec-4d2c-a9ef-7a4b6d64e8b2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f26a346-5d99-4526-bdbe-663add0ee26b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing2)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12aa1be1-70b5-42ac-90ef-7329c9969c13",
   "metadata": {},
   "source": [
    "# keep training df5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed94969e-53d4-4710-8956-3341d97338cb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataframes = [df5]\n",
    "\n",
    "# Load the last saved model\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_4')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_4')\n",
    "\n",
    "# Continue training loop over the next set of dataframes\n",
    "for idx, df in enumerate(dataframes, start=2):  # Start index from 2 to continue numbering\n",
    "    print(f\"Training on dataframe {idx+1}/{len(dataframes)+2}\")\n",
    "    \n",
    "    # Split dataframe into train, validation, and test sets\n",
    "    df_train, df_test = train_test_split(df, stratify=df['label'], test_size=0.2, random_state=42)\n",
    "    df_train, df_val = train_test_split(df_train, stratify=df_train['label'], test_size=0.2, random_state=42)\n",
    "    \n",
    "    # Convert to datasets\n",
    "    dataset_train = Dataset.from_pandas(df_train)\n",
    "    dataset_val = Dataset.from_pandas(df_val)\n",
    "    dataset_test = Dataset.from_pandas(df_test)\n",
    "    \n",
    "    # Tokenize datasets\n",
    "    dataset_train = dataset_train.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_val = dataset_val.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    \n",
    "    # Set format for PyTorch\n",
    "    dataset_train.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_val.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    \n",
    "    # Initialize Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=args,\n",
    "        train_dataset=dataset_train,\n",
    "        eval_dataset=dataset_val,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "    \n",
    "    # Train the model\n",
    "    trainer.train()\n",
    "    \n",
    "    # Save the model after each dataframe\n",
    "    model.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    tokenizer.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    \n",
    "    # Optionally, evaluate on the test set after each dataframe\n",
    "    eval_results = trainer.evaluate(eval_dataset=dataset_test)\n",
    "    print(f\"Eval results for dataframe {idx+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e814052-f72a-4fb5-8e15-f34a7ebfdc74",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.save_pretrained(f'temp/model_df_5')\n",
    "tokenizer.save_pretrained(f'temp/model_df_5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f34d843b-56f1-434c-80c9-793222aaa0b8",
   "metadata": {},
   "source": [
    "# try using model 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61fb00da-c8a0-47e5-9141-b2da504245ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the saved model and tokenizer\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_5')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34ad3c69-b9ec-4340-a526-43f70c1fd277",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "781dfb39-f244-4f69-b100-b08e5b1ce9c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing2)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "baaaa4d7-2ec5-4dc2-8a3e-2bb2fbcaeaad",
   "metadata": {},
   "source": [
    "# keep training df6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "023ae897-d4c8-489c-b222-41db3a3ad742",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataframes = [df6]\n",
    "\n",
    "# Load the last saved model\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_5')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_5')\n",
    "\n",
    "# Continue training loop over the next set of dataframes\n",
    "for idx, df in enumerate(dataframes, start=2):  # Start index from 2 to continue numbering\n",
    "    print(f\"Training on dataframe {idx+1}/{len(dataframes)+2}\")\n",
    "    \n",
    "    # Split dataframe into train, validation, and test sets\n",
    "    df_train, df_test = train_test_split(df, stratify=df['label'], test_size=0.2, random_state=42)\n",
    "    df_train, df_val = train_test_split(df_train, stratify=df_train['label'], test_size=0.2, random_state=42)\n",
    "    \n",
    "    # Convert to datasets\n",
    "    dataset_train = Dataset.from_pandas(df_train)\n",
    "    dataset_val = Dataset.from_pandas(df_val)\n",
    "    dataset_test = Dataset.from_pandas(df_test)\n",
    "    \n",
    "    # Tokenize datasets\n",
    "    dataset_train = dataset_train.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_val = dataset_val.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    \n",
    "    # Set format for PyTorch\n",
    "    dataset_train.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_val.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    \n",
    "    # Initialize Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=args,\n",
    "        train_dataset=dataset_train,\n",
    "        eval_dataset=dataset_val,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "    \n",
    "    # Train the model\n",
    "    trainer.train()\n",
    "    \n",
    "    # Save the model after each dataframe\n",
    "    model.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    tokenizer.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    \n",
    "    # Optionally, evaluate on the test set after each dataframe\n",
    "    eval_results = trainer.evaluate(eval_dataset=dataset_test)\n",
    "    print(f\"Eval results for dataframe {idx+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2783f03a-7e52-4316-9738-a15213c4f65d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.save_pretrained(f'temp/model_df_6')\n",
    "tokenizer.save_pretrained(f'temp/model_df_6')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8cc410d-b876-4f86-9f74-7a15fbed2d0c",
   "metadata": {},
   "source": [
    "# try using model 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9f3d8d9-5edf-4b40-b2cf-b70a9ced4eff",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load the saved model and tokenizer\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_6')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_6')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45a1f667-e826-44bb-83da-14c23d7cfec3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ae826dd-682a-414e-9cb8-edc3c0076fb5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing2)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10d3c29a-2912-4d1c-b5fe-223cf3b1844d",
   "metadata": {
    "tags": []
   },
   "source": [
    "# keep training df7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "adeafea6-3236-47f6-99fe-4be506d9f9de",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on dataframe 3/3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6727a02fb60d4528b13b2d74791bcc2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/3738 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1449c0bc67664bada140af2d9089e4e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/935 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da9f917fb693463ebeb2bceb6a0e4f9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1169 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1404' max='1404' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1404/1404 05:00, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.377300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.207800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for dataframe 3: {'eval_loss': 0.37617334723472595, 'eval_accuracy': 0.8665526090675791, 'eval_runtime': 7.8681, 'eval_samples_per_second': 148.575, 'eval_steps_per_second': 4.703, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "dataframes = [df7]\n",
    "\n",
    "# Load the last saved model\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_6')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_6')\n",
    "\n",
    "# Continue training loop over the next set of dataframes\n",
    "for idx, df in enumerate(dataframes, start=2):  # Start index from 2 to continue numbering\n",
    "    print(f\"Training on dataframe {idx+1}/{len(dataframes)+2}\")\n",
    "    \n",
    "    # Split dataframe into train, validation, and test sets\n",
    "    df_train, df_test = train_test_split(df, stratify=df['label'], test_size=0.2, random_state=42)\n",
    "    df_train, df_val = train_test_split(df_train, stratify=df_train['label'], test_size=0.2, random_state=42)\n",
    "    \n",
    "    # Convert to datasets\n",
    "    dataset_train = Dataset.from_pandas(df_train)\n",
    "    dataset_val = Dataset.from_pandas(df_val)\n",
    "    dataset_test = Dataset.from_pandas(df_test)\n",
    "    \n",
    "    # Tokenize datasets\n",
    "    dataset_train = dataset_train.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_val = dataset_val.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    \n",
    "    # Set format for PyTorch\n",
    "    dataset_train.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_val.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    \n",
    "    # Initialize Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=args,\n",
    "        train_dataset=dataset_train,\n",
    "        eval_dataset=dataset_val,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "    \n",
    "    # Train the model\n",
    "    trainer.train()\n",
    "    \n",
    "    # Save the model after each dataframe\n",
    "    model.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    tokenizer.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    \n",
    "    # Optionally, evaluate on the test set after each dataframe\n",
    "    eval_results = trainer.evaluate(eval_dataset=dataset_test)\n",
    "    print(f\"Eval results for dataframe {idx+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "59bdc72d-2191-47f8-b34c-158cf0b092ab",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('temp/model_df_7/tokenizer_config.json',\n",
       " 'temp/model_df_7/special_tokens_map.json',\n",
       " 'temp/model_df_7/vocab.txt',\n",
       " 'temp/model_df_7/added_tokens.json')"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_pretrained(f'temp/model_df_7')\n",
    "tokenizer.save_pretrained(f'temp/model_df_7')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "39c4d4c6-103a-45db-9d4a-72c81205d023",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load the saved model and tokenizer\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_7')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_7')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "a677f1e3-5a27-4d64-a4d3-9714c88ed130",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1b0fa47b14c4d0589d3a6bf1dc7175b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1169 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results: {'eval_loss': 0.17151209712028503, 'eval_accuracy': 0.9084687767322498, 'eval_runtime': 7.7511, 'eval_samples_per_second': 150.816, 'eval_steps_per_second': 4.773}\n"
     ]
    }
   ],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing2)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9bb0951-9864-4ea5-b65c-7b14e6fc4fce",
   "metadata": {},
   "source": [
    "# keep training df8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "b88470f5-8a1c-4455-bd4f-1a56f3a2054d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on dataframe 3/3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fbcec6b4b447481eaa505f107a2a3207",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/614 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd76f86aedd54541895e4df8cfa43df8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/154 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3d51d69bb41437b81f333efbc59f7fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/193 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='231' max='231' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [231/231 00:47, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7/7 00:01]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for dataframe 3: {'eval_loss': 0.27275797724723816, 'eval_accuracy': 0.9481865284974094, 'eval_runtime': 1.3352, 'eval_samples_per_second': 144.55, 'eval_steps_per_second': 5.243, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "dataframes = [df8]\n",
    "\n",
    "# Load the last saved model\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_7')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_7')\n",
    "\n",
    "# Continue training loop over the next set of dataframes\n",
    "for idx, df in enumerate(dataframes, start=2):  # Start index from 2 to continue numbering\n",
    "    print(f\"Training on dataframe {idx+1}/{len(dataframes)+2}\")\n",
    "    \n",
    "    # Split dataframe into train, validation, and test sets\n",
    "    df_train, df_test = train_test_split(df, stratify=df['label'], test_size=0.2, random_state=42)\n",
    "    df_train, df_val = train_test_split(df_train, stratify=df_train['label'], test_size=0.2, random_state=42)\n",
    "    \n",
    "    # Convert to datasets\n",
    "    dataset_train = Dataset.from_pandas(df_train)\n",
    "    dataset_val = Dataset.from_pandas(df_val)\n",
    "    dataset_test = Dataset.from_pandas(df_test)\n",
    "    \n",
    "    # Tokenize datasets\n",
    "    dataset_train = dataset_train.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_val = dataset_val.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "    \n",
    "    # Set format for PyTorch\n",
    "    dataset_train.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_val.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "    \n",
    "    # Initialize Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=args,\n",
    "        train_dataset=dataset_train,\n",
    "        eval_dataset=dataset_val,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "    \n",
    "    # Train the model\n",
    "    trainer.train()\n",
    "    \n",
    "    # Save the model after each dataframe\n",
    "    model.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    tokenizer.save_pretrained(f'temp/model_df_{idx+1}')\n",
    "    \n",
    "    # Optionally, evaluate on the test set after each dataframe\n",
    "    eval_results = trainer.evaluate(eval_dataset=dataset_test)\n",
    "    print(f\"Eval results for dataframe {idx+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "b1b8bc20-ecee-49e3-8023-33fa7f9881aa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('temp/model_df_8/tokenizer_config.json',\n",
       " 'temp/model_df_8/special_tokens_map.json',\n",
       " 'temp/model_df_8/vocab.txt',\n",
       " 'temp/model_df_8/added_tokens.json')"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_pretrained(f'temp/model_df_8')\n",
    "tokenizer.save_pretrained(f'temp/model_df_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "34be1433-8047-47f5-bb0d-800add8dfcad",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load the saved model and tokenizer\n",
    "model = BertForSequenceClassification.from_pretrained('temp/model_df_8')\n",
    "tokenizer = BertTokenizer.from_pretrained('temp/model_df_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "5a2ef8ea-ebba-413e-967a-6a4c3e405367",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbcf839e269f4b9faaf2010f0ab031d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1169 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results: {'eval_loss': 0.3001035153865814, 'eval_accuracy': 0.8964927288280582, 'eval_runtime': 7.567, 'eval_samples_per_second': 154.486, 'eval_steps_per_second': 4.89}\n"
     ]
    }
   ],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing2)\n",
    "\n",
    "# Tokenize the test dataset\n",
    "dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define compute metrics function\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return {'accuracy': accuracy_score(predictions, labels)}\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    eval_dataset=dataset_test,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Eval results: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e78cf0c4-6565-4192-b08a-9912b9c7714d",
   "metadata": {},
   "source": [
    "# Evaluate overall models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "3232d9ea-6125-47bf-83eb-71819fcbe2e7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b54bf0b3c25346b5912ac6aa8b7f3382",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1169 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for temp/model_df_1: {'eval_loss': 0.4753454625606537, 'eval_accuracy': 0.8887938408896493, 'eval_precision': 0.91685506028011, 'eval_recall': 0.8887938408896493, 'eval_f1': 0.8945557132392937, 'eval_runtime': 8.018, 'eval_samples_per_second': 145.798, 'eval_steps_per_second': 4.615}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:08]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for temp/model_df_2: {'eval_loss': 0.4671970307826996, 'eval_accuracy': 0.8922155688622755, 'eval_precision': 0.9160435261539863, 'eval_recall': 0.8922155688622755, 'eval_f1': 0.8974479692377935, 'eval_runtime': 8.3101, 'eval_samples_per_second': 140.672, 'eval_steps_per_second': 4.452}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:08]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for temp/model_df_3: {'eval_loss': 0.3001035153865814, 'eval_accuracy': 0.8964927288280582, 'eval_precision': 0.9251385140263982, 'eval_recall': 0.8964927288280582, 'eval_f1': 0.9009000197422004, 'eval_runtime': 8.4479, 'eval_samples_per_second': 138.378, 'eval_steps_per_second': 4.38}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for temp/model_df_4: {'eval_loss': 1.4409071207046509, 'eval_accuracy': 0.7750213857998289, 'eval_precision': 0.8011277843465486, 'eval_recall': 0.7750213857998289, 'eval_f1': 0.7769965119923422, 'eval_runtime': 8.2165, 'eval_samples_per_second': 142.275, 'eval_steps_per_second': 4.503}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for temp/model_df_5: {'eval_loss': 1.1273574829101562, 'eval_accuracy': 0.853721129170231, 'eval_precision': 0.8788014041491926, 'eval_recall': 0.853721129170231, 'eval_f1': 0.8602961162053503, 'eval_runtime': 7.9587, 'eval_samples_per_second': 146.883, 'eval_steps_per_second': 4.649}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for temp/model_df_6: {'eval_loss': 0.8956769704818726, 'eval_accuracy': 0.8383233532934131, 'eval_precision': 0.8704024996943253, 'eval_recall': 0.8383233532934131, 'eval_f1': 0.8463496710883314, 'eval_runtime': 7.7943, 'eval_samples_per_second': 149.981, 'eval_steps_per_second': 4.747}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for temp/model_df_7: {'eval_loss': 0.17151209712028503, 'eval_accuracy': 0.9084687767322498, 'eval_precision': 0.9277880217020845, 'eval_recall': 0.9084687767322498, 'eval_f1': 0.9132672342615976, 'eval_runtime': 7.6839, 'eval_samples_per_second': 152.137, 'eval_steps_per_second': 4.815}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [37/37 00:07]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval results for temp/model_df_8: {'eval_loss': 0.3001035153865814, 'eval_accuracy': 0.8964927288280582, 'eval_precision': 0.9251385140263982, 'eval_recall': 0.8964927288280582, 'eval_f1': 0.9009000197422004, 'eval_runtime': 7.6205, 'eval_samples_per_second': 153.401, 'eval_steps_per_second': 4.855}\n",
      "                 eval_loss  eval_accuracy  eval_precision  eval_recall  \\\n",
      "temp/model_df_1   0.475345       0.888794        0.916855     0.888794   \n",
      "temp/model_df_2   0.467197       0.892216        0.916044     0.892216   \n",
      "temp/model_df_3   0.300104       0.896493        0.925139     0.896493   \n",
      "temp/model_df_4   1.440907       0.775021        0.801128     0.775021   \n",
      "temp/model_df_5   1.127357       0.853721        0.878801     0.853721   \n",
      "temp/model_df_6   0.895677       0.838323        0.870402     0.838323   \n",
      "temp/model_df_7   0.171512       0.908469        0.927788     0.908469   \n",
      "temp/model_df_8   0.300104       0.896493        0.925139     0.896493   \n",
      "\n",
      "                  eval_f1  eval_runtime  eval_samples_per_second  \\\n",
      "temp/model_df_1  0.894556        8.0180                  145.798   \n",
      "temp/model_df_2  0.897448        8.3101                  140.672   \n",
      "temp/model_df_3  0.900900        8.4479                  138.378   \n",
      "temp/model_df_4  0.776997        8.2165                  142.275   \n",
      "temp/model_df_5  0.860296        7.9587                  146.883   \n",
      "temp/model_df_6  0.846350        7.7943                  149.981   \n",
      "temp/model_df_7  0.913267        7.6839                  152.137   \n",
      "temp/model_df_8  0.900900        7.6205                  153.401   \n",
      "\n",
      "                 eval_steps_per_second  \n",
      "temp/model_df_1                  4.615  \n",
      "temp/model_df_2                  4.452  \n",
      "temp/model_df_3                  4.380  \n",
      "temp/model_df_4                  4.503  \n",
      "temp/model_df_5                  4.649  \n",
      "temp/model_df_6                  4.747  \n",
      "temp/model_df_7                  4.815  \n",
      "temp/model_df_8                  4.855  \n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, BertTokenizer, Trainer, TrainingArguments\n",
    "from datasets import Dataset\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
    "import pandas as pd\n",
    "\n",
    "# Load the test dataset\n",
    "dataset_test = Dataset.from_pandas(df_testing2)\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, predictions, average='weighted')\n",
    "    accuracy = accuracy_score(labels, predictions)\n",
    "    return {\n",
    "        'accuracy': accuracy,\n",
    "        'precision': precision,\n",
    "        'recall': recall,\n",
    "        'f1': f1\n",
    "    }\n",
    "\n",
    "# Define training arguments for evaluation\n",
    "args = TrainingArguments(\n",
    "    output_dir='temp/',\n",
    "    per_device_eval_batch_size=32,\n",
    ")\n",
    "\n",
    "# List of model checkpoints\n",
    "model_checkpoints = [f'temp/model_df_{i}' for i in range(1, 9)]\n",
    "\n",
    "# Tokenize the test dataset once\n",
    "tokenizer = BertTokenizer.from_pretrained(model_checkpoints[0])\n",
    "tokenized_dataset_test = dataset_test.map(lambda e: tokenizer(e['text'], truncation=True, padding='max_length', max_length=128), batched=True)\n",
    "tokenized_dataset_test.set_format(type='torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Dictionary to store evaluation results\n",
    "eval_results_dict = {}\n",
    "\n",
    "for checkpoint in model_checkpoints:\n",
    "    # Load the saved model\n",
    "    model = BertForSequenceClassification.from_pretrained(checkpoint)\n",
    "    \n",
    "    # Initialize Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=args,\n",
    "        eval_dataset=tokenized_dataset_test,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "    \n",
    "    # Evaluate the model\n",
    "    eval_results = trainer.evaluate()\n",
    "    eval_results_dict[checkpoint] = eval_results\n",
    "    print(f\"Eval results for {checkpoint}: {eval_results}\")\n",
    "\n",
    "eval_results_df = pd.DataFrame(eval_results_dict).T\n",
    "print(eval_results_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "947ddfe3-5552-48b9-b956-62300e3da54e",
   "metadata": {},
   "source": [
    "# Compare before and after fine tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "44d52667-3654-4609-a842-3210b17a538b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at yiyanghkust/finbert-pretrain and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fine-tuned Classifier Weights:\n",
      "Parameter containing:\n",
      "tensor([[-0.0198, -0.0209,  0.0135,  ...,  0.0032, -0.0268, -0.0092],\n",
      "        [-0.0038, -0.0017,  0.0402,  ...,  0.0201,  0.0100,  0.0358],\n",
      "        [-0.0035,  0.0289,  0.0178,  ...,  0.0109,  0.0106, -0.0021]],\n",
      "       requires_grad=True)\n",
      "Pre-trained Classifier Weights:\n",
      "Parameter containing:\n",
      "tensor([[-0.0099,  0.0073,  0.0068,  ..., -0.0317,  0.0128, -0.0054],\n",
      "        [-0.0122,  0.0022,  0.0025,  ...,  0.0041,  0.0189, -0.0233],\n",
      "        [ 0.0128,  0.0216, -0.0242,  ...,  0.0176, -0.0040,  0.0109]],\n",
      "       requires_grad=True)\n",
      "Fine-tuned Classifier Bias:\n",
      "Parameter containing:\n",
      "tensor([ 0.0011, -0.0011, -0.0017], requires_grad=True)\n",
      "Pre-trained Classifier Bias:\n",
      "Parameter containing:\n",
      "tensor([0., 0., 0.], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# Load the fine-tuned model\n",
    "fine_tuned_model = BertForSequenceClassification.from_pretrained('temp/model_df_7')\n",
    "\n",
    "# Load the pre-trained model\n",
    "pretrained_model = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-pretrain', num_labels=3)\n",
    "\n",
    "# Access classifier weights and biases\n",
    "fine_tuned_classifier_weights = fine_tuned_model.classifier.weight\n",
    "fine_tuned_classifier_bias = fine_tuned_model.classifier.bias\n",
    "\n",
    "pretrained_classifier_weights = pretrained_model.classifier.weight\n",
    "pretrained_classifier_bias = pretrained_model.classifier.bias\n",
    "\n",
    "print(\"Fine-tuned Classifier Weights:\")\n",
    "print(fine_tuned_classifier_weights)\n",
    "\n",
    "print(\"Pre-trained Classifier Weights:\")\n",
    "print(pretrained_classifier_weights)\n",
    "\n",
    "print(\"Fine-tuned Classifier Bias:\")\n",
    "print(fine_tuned_classifier_bias)\n",
    "\n",
    "print(\"Pre-trained Classifier Bias:\")\n",
    "print(pretrained_classifier_bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "120666b0-ac56-4366-aa5a-40adc4e9f546",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fine-tuned Encoder Query Weights (Layer 0):\n",
      "Parameter containing:\n",
      "tensor([[-5.6009e-02,  3.2049e-02, -5.5241e-03,  ...,  2.2374e-02,\n",
      "          1.1440e-02,  5.3082e-03],\n",
      "        [ 3.0205e-02,  3.7393e-02,  2.0345e-02,  ...,  1.9946e-02,\n",
      "          1.6365e-03,  1.1028e-02],\n",
      "        [ 6.2553e-03, -3.2888e-02, -4.2043e-03,  ..., -1.5961e-02,\n",
      "         -3.6810e-02, -1.8867e-02],\n",
      "        ...,\n",
      "        [-1.1099e-02, -3.8351e-02,  1.4641e-02,  ...,  2.4120e-03,\n",
      "         -9.5471e-03, -4.5184e-03],\n",
      "        [-2.1598e-03,  6.1430e-03,  3.0534e-05,  ...,  2.0336e-02,\n",
      "          1.0604e-02,  1.4497e-02],\n",
      "        [ 3.3183e-03,  1.6679e-02, -1.2548e-02,  ..., -1.1251e-02,\n",
      "         -4.8407e-02,  1.7429e-02]], requires_grad=True)\n",
      "Pre-trained Encoder Query Weights (Layer 0):\n",
      "Parameter containing:\n",
      "tensor([[-0.0496,  0.0298, -0.0069,  ...,  0.0254,  0.0102,  0.0027],\n",
      "        [ 0.0254,  0.0388,  0.0234,  ...,  0.0176,  0.0007,  0.0110],\n",
      "        [ 0.0097, -0.0304, -0.0069,  ..., -0.0147, -0.0396, -0.0197],\n",
      "        ...,\n",
      "        [-0.0120, -0.0398,  0.0151,  ..., -0.0023, -0.0113, -0.0062],\n",
      "        [-0.0034,  0.0066, -0.0004,  ...,  0.0208,  0.0105,  0.0132],\n",
      "        [ 0.0068,  0.0135, -0.0128,  ..., -0.0130, -0.0460,  0.0190]],\n",
      "       requires_grad=True)\n",
      "Fine-tuned Encoder Key Weights (Layer 0):\n",
      "Parameter containing:\n",
      "tensor([[-0.0074, -0.0358,  0.0462,  ...,  0.0180, -0.0288,  0.0021],\n",
      "        [-0.0087, -0.0230, -0.0517,  ...,  0.0272,  0.0188, -0.0133],\n",
      "        [-0.0118, -0.0006,  0.0103,  ..., -0.0030, -0.0115,  0.0123],\n",
      "        ...,\n",
      "        [-0.0236,  0.0093, -0.0357,  ..., -0.0186,  0.0147,  0.0259],\n",
      "        [ 0.0064,  0.0288,  0.0144,  ...,  0.0097,  0.0459,  0.0388],\n",
      "        [ 0.0204, -0.0239, -0.0336,  ...,  0.0223,  0.0393, -0.0386]],\n",
      "       requires_grad=True)\n",
      "Pre-trained Encoder Key Weights (Layer 0):\n",
      "Parameter containing:\n",
      "tensor([[-0.0066, -0.0376,  0.0487,  ...,  0.0169, -0.0294,  0.0026],\n",
      "        [-0.0058, -0.0215, -0.0528,  ...,  0.0262,  0.0147, -0.0134],\n",
      "        [-0.0117,  0.0016,  0.0111,  ...,  0.0006, -0.0079,  0.0088],\n",
      "        ...,\n",
      "        [-0.0245,  0.0115, -0.0319,  ..., -0.0194,  0.0147,  0.0257],\n",
      "        [ 0.0075,  0.0256,  0.0130,  ...,  0.0084,  0.0431,  0.0344],\n",
      "        [ 0.0178, -0.0200, -0.0299,  ...,  0.0189,  0.0403, -0.0370]],\n",
      "       requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "fine_tuned_encoder_layer = fine_tuned_model.bert.encoder.layer[0]\n",
    "pretrained_encoder_layer = pretrained_model.bert.encoder.layer[0]\n",
    "\n",
    "fine_tuned_query_weights = fine_tuned_encoder_layer.attention.self.query.weight\n",
    "pretrained_query_weights = pretrained_encoder_layer.attention.self.query.weight\n",
    "\n",
    "print(\"Fine-tuned Encoder Query Weights (Layer 0):\")\n",
    "print(fine_tuned_query_weights)\n",
    "\n",
    "print(\"Pre-trained Encoder Query Weights (Layer 0):\")\n",
    "print(pretrained_query_weights)\n",
    "\n",
    "fine_tuned_key_weights = fine_tuned_encoder_layer.attention.self.key.weight\n",
    "pretrained_key_weights = pretrained_encoder_layer.attention.self.key.weight\n",
    "\n",
    "print(\"Fine-tuned Encoder Key Weights (Layer 0):\")\n",
    "print(fine_tuned_key_weights)\n",
    "\n",
    "print(\"Pre-trained Encoder Key Weights (Layer 0):\")\n",
    "print(pretrained_key_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "5134c1f0-936d-4b77-be6d-7c09e1fa9bd6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fine-tuned Classifier Weights Summary:\n",
      "{'mean': 5.3044414016767405e-06, 'std': 0.01976761408150196, 'norm': 0.9486396312713623}\n",
      "Pre-trained Classifier Weights Summary:\n",
      "{'mean': 0.0003834100207313895, 'std': 0.02015182375907898, 'norm': 0.9672526717185974}\n",
      "Fine-tuned Encoder Query Weights Summary (Layer 0):\n",
      "{'mean': 4.8687521484680474e-05, 'std': 0.025672325864434242, 'norm': 19.716251373291016}\n",
      "Pre-trained Encoder Query Weights Summary (Layer 0):\n",
      "{'mean': 4.6587814722443e-05, 'std': 0.025565769523382187, 'norm': 19.63439178466797}\n"
     ]
    }
   ],
   "source": [
    "# Calculate statistics for classifier weights\n",
    "def summarize_weights(weights):\n",
    "    return {\n",
    "        \"mean\": torch.mean(weights).item(),\n",
    "        \"std\": torch.std(weights).item(),\n",
    "        \"norm\": torch.norm(weights).item()\n",
    "    }\n",
    "\n",
    "fine_tuned_classifier_summary = summarize_weights(fine_tuned_classifier_weights)\n",
    "pretrained_classifier_summary = summarize_weights(pretrained_classifier_weights)\n",
    "\n",
    "print(\"Fine-tuned Classifier Weights Summary:\")\n",
    "print(fine_tuned_classifier_summary)\n",
    "\n",
    "print(\"Pre-trained Classifier Weights Summary:\")\n",
    "print(pretrained_classifier_summary)\n",
    "\n",
    "fine_tuned_query_summary = summarize_weights(fine_tuned_query_weights)\n",
    "pretrained_query_summary = summarize_weights(pretrained_query_weights)\n",
    "\n",
    "print(\"Fine-tuned Encoder Query Weights Summary (Layer 0):\")\n",
    "print(fine_tuned_query_summary)\n",
    "\n",
    "print(\"Pre-trained Encoder Query Weights Summary (Layer 0):\")\n",
    "print(pretrained_query_summary)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "a9e39a93-2039-48d0-8f16-a9d143bb8cc4",
   "metadata": {},
   "source": [
    "Classifier Layer: The classifier weights have undergone more noticeable changes in their mean values due to fine-tuning, reflecting adjustments to better fit the new data.\n",
    "Encoder Layer: The encoder query weights in the first layer show minimal changes in their statistical properties, suggesting that while fine-tuning has made adjustments, the overall structure and distribution of these weights have remained quite stable."
   ]
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "conda-base-py",
   "name": "workbench-notebooks.m121",
   "type": "gcloud",
   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m121"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel) (Local)",
   "language": "python",
   "name": "conda-base-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}