generativeAI_project2A.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "OHqQt27bR0dg"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn\n",
    "import torch.nn.utils.rnn\n",
    "import torch.utils.data\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 370
    },
    "id": "we4PPrJ-T3-g",
    "outputId": "ab249b93-493a-4ffc-bfb6-3ff1c1fbb472"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2632500 entries, 0 to 2632499\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Dtype \n",
      "---  ------  ----- \n",
      " 0   src     object\n",
      " 1   tgt     int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 40.2+ MB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>tgt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0+0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0-0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0*0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(0+0)*0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0+0*0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        src  tgt\n",
       "0      0+0=    0\n",
       "1      0-0=    0\n",
       "2      0*0=    0\n",
       "3  (0+0)*0=    0\n",
       "4    0+0*0=    0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('arithmetic.csv')\n",
    "df.info()\n",
    "# 看一下前幾筆資料是什麼樣子\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fY2NI3XissdK"
   },
   "source": [
    "# 建立字典\n",
    "- 無法直接利用純文字進行計算\n",
    "- 將所有文字轉換成數字\n",
    "- 字典大小約為 `7000`\n",
    "- 特殊字\n",
    "    - '&lt;pad&gt;'\n",
    "        - 每個 batch 所包含的句子長度不同\n",
    "        - 將長度使用 '&lt;pad&gt;' 補成 batch 中最大值者\n",
    "    - '&lt;eos&gt;'\n",
    "        - 指定生成的結尾\n",
    "        - 沒有 '&lt;eos&gt;' 會不知道何時停止生成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Em6Hod-AssdL",
    "outputId": "08016904-3dab-41f9-e01a-9ac300f19abd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "字典大小: 18\n"
     ]
    }
   ],
   "source": [
    "# 一個dict把中文字符轉化成id\n",
    "char_to_id = {}\n",
    "# 把id轉回中文字符\n",
    "id_to_char = {}\n",
    "\n",
    "# 有一些必須要用的special token先添加進來(一般用來做padding的token的id是0)\n",
    "char_to_id['<pad>'] = 0\n",
    "char_to_id['<eos>'] = 1\n",
    "id_to_char[0] = '<pad>'\n",
    "id_to_char[1] = '<eos>'\n",
    "\n",
    "# 把所有資料集中出現的token都記錄到dict中\n",
    "for char in set(df['src'].str.cat()):\n",
    "    ch_id = len(char_to_id)\n",
    "    char_to_id[char] = ch_id\n",
    "    id_to_char[ch_id] = char\n",
    "\n",
    "vocab_size = len(char_to_id)\n",
    "print('字典大小: {}'.format(vocab_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset 1: 只留下加減法資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>tgt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0+0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0-0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0+0+0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0-0-0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0+0-0=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       src  tgt\n",
       "0     0+0=    0\n",
       "1     0-0=    0\n",
       "15  0+0+0=    0\n",
       "16  0-0-0=    0\n",
       "18  0+0-0=    0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df = df[~df['src'].str.contains(\"\\*\")]\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "xZDW6Zmz2RcC"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_9064/958800755.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  new_df['src'] = new_df['src'].apply(lambda text: [char_to_id[char] for char in text])\n",
      "/tmp/ipykernel_9064/958800755.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  new_df['tgt'] = new_df['tgt'].apply(lambda num: [char_to_id[char] for char in str(num)])\n"
     ]
    }
   ],
   "source": [
    "# 把資料轉成id\n",
    "new_df['src'] = new_df['src'].apply(lambda text: [char_to_id[char] for char in text])\n",
    "new_df['tgt'] = new_df['tgt'].apply(lambda num: [char_to_id[char] for char in str(num)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把資料分成train, val, test = 0.8 : 0.1 : 0.1\n",
    "train_data, val_data = train_test_split(new_df, test_size=0.2, random_state=250)\n",
    "val_data, test_data = train_test_split(val_data, test_size=0.5, random_state=250)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1Q8LAMUBssdL"
   },
   "source": [
    "# 超參數\n",
    "\n",
    "|超參數|意義|\n",
    "|-|-|\n",
    "|`batch_size`|單一 batch 的資料數|\n",
    "|`epochs`|總共要訓練幾個 epoch|\n",
    "|`embed_dim`|文字的 embedding 維度|\n",
    "|`hidden_dim`|RNN 中每個時間的 hidden state 維度|\n",
    "|`lr`|Learning Rate|\n",
    "|`grad_clip`|為了避免 RNN 出現梯度爆炸問題，將梯度限制範圍|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "mlpbhW8weuvg"
   },
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "epochs = 5\n",
    "embed_dim = 256\n",
    "hidden_dim = 256\n",
    "lr = 0.001\n",
    "grad_clip = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "r15x449HssdM"
   },
   "source": [
    "# 資料分批\n",
    "- 使用 `torch.utils.data.Dataset` 建立資料產生的工具 `dataset`\n",
    "- 再使用 `torch.utils.data.DataLoader` 對資料集 `dataset` 隨機抽樣並作為一個 batch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "xSVqB7USssdM"
   },
   "outputs": [],
   "source": [
    "# 這裏的dataset是文本生成的dataset，輸入和輸出的資料都是文章\n",
    "# 舉個例子，現在的狀況是：\n",
    "# input:  A B C D E F\n",
    "# output: B C D E F <eos>\n",
    "# 而對於加減法的任務：\n",
    "# input:  1 + 2 + 3 = 6\n",
    "# output: / / / / / 6 <eos>\n",
    "# /的部分都不用算loss，主要是預測=的後面，這裏的答案是6，所以output是6 <eos>\n",
    "\n",
    "\n",
    "class Dataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, sequences):\n",
    "        self.sequences = sequences\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        y = self.sequences['tgt'].iloc[index]\n",
    "        x = self.sequences['src'].iloc[index] + [char_to_id['<pad>']] * len(y)\n",
    "        y = self.sequences['tgt'].iloc[index] + [char_to_id['<eos>']]\n",
    "        y = [char_to_id['<pad>']] * (len(x) - len(y)) + y\n",
    "        return x, y\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.sequences)\n",
    "\n",
    "\n",
    "def collate_fn(batch):\n",
    "    batch_x = [torch.tensor(data[0]) for data in batch] # list[torch.tensor]\n",
    "    batch_y = [torch.tensor(data[1]) for data in batch] # list[torch.tensor]\n",
    "    batch_x_lens = torch.LongTensor([len(x) for x in batch_x])\n",
    "    batch_y_lens = torch.LongTensor([len(y) for y in batch_y])\n",
    "\n",
    "    # torch.tensor\n",
    "    # [[1968, 1891, 3580, ... , 0, 0, 0],\n",
    "    #  [1014, 2242, 2247, ... , 0, 0, 0],\n",
    "    #  [3032,  522, 1485, ... , 0, 0, 0]]\n",
    "    #                       padding↑\n",
    "    pad_batch_x = torch.nn.utils.rnn.pad_sequence(batch_x,\n",
    "                                                  batch_first=True, # shape=(batch_size, seq_len)\n",
    "                                                  padding_value=char_to_id['<pad>'])\n",
    "\n",
    "    pad_batch_y = torch.nn.utils.rnn.pad_sequence(batch_y,\n",
    "                                                  batch_first=True, # shape=(batch_size, seq_len)\n",
    "                                                  padding_value=char_to_id['<pad>'])\n",
    "\n",
    "    return pad_batch_x, pad_batch_y, batch_x_lens, batch_y_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "Rd4bWLP1ssdM"
   },
   "outputs": [],
   "source": [
    "dataset_train = Dataset(train_data)\n",
    "dataset_val = Dataset(val_data)\n",
    "dataset_test = Dataset(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "DYp9q3e5ssdM"
   },
   "outputs": [],
   "source": [
    "data_loader_train_64 = torch.utils.data.DataLoader(dataset_train,\n",
    "                                                batch_size=batch_size,\n",
    "                                                shuffle=True,\n",
    "                                                collate_fn=collate_fn)\n",
    "\n",
    "data_loader_val_64 = torch.utils.data.DataLoader(dataset_val,\n",
    "                                                batch_size=batch_size,\n",
    "                                                shuffle=True,\n",
    "                                                collate_fn=collate_fn)\n",
    "\n",
    "data_loader_test_64 = torch.utils.data.DataLoader(dataset_test,\n",
    "                                                batch_size=batch_size,\n",
    "                                                shuffle=True,\n",
    "                                                collate_fn=collate_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "DYp9q3e5ssdM"
   },
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "\n",
    "data_loader_train_128 = torch.utils.data.DataLoader(dataset_train,\n",
    "                                                batch_size=batch_size,\n",
    "                                                shuffle=True,\n",
    "                                                collate_fn=collate_fn)\n",
    "\n",
    "data_loader_val_128 = torch.utils.data.DataLoader(dataset_val,\n",
    "                                                batch_size=batch_size,\n",
    "                                                shuffle=True,\n",
    "                                                collate_fn=collate_fn)\n",
    "\n",
    "data_loader_test_128 = torch.utils.data.DataLoader(dataset_test,\n",
    "                                                batch_size=batch_size,\n",
    "                                                shuffle=True,\n",
    "                                                collate_fn=collate_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KWuAvp-ZssdM",
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# 模型設計\n",
    "\n",
    "## 執行順序\n",
    "1. 將句子中的所有字轉換成 embedding\n",
    "2. 按照句子順序將 embedding 丟入 RNN\n",
    "3. RNN 的輸出再丟給 RNN，可以接上更多層\n",
    "4. 最後的 RNN 所有時間點的輸出丟進一層 Fully Connected\n",
    "5. 輸出結果所有維度中的最大者即為下一個字\n",
    "\n",
    "## 損失函數\n",
    "因為是類別預測，所以使用 Cross Entropy\n",
    "\n",
    "## 梯度更新\n",
    "使用 Adam 演算法進行梯度更新"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "fNhof8yVssdM"
   },
   "outputs": [],
   "source": [
    "class CharRNN(torch.nn.Module):\n",
    "    def __init__(self, vocab_size, embed_dim, hidden_dim):\n",
    "        super(CharRNN, self).__init__()\n",
    "\n",
    "        # Embedding層\n",
    "        self.embedding = torch.nn.Embedding(num_embeddings=vocab_size,\n",
    "                                            embedding_dim=embed_dim,\n",
    "                                            padding_idx=char_to_id['<pad>'])\n",
    "\n",
    "        # RNN層\n",
    "        self.rnn_layer1 = torch.nn.RNN(input_size=embed_dim,\n",
    "                                        hidden_size=hidden_dim,\n",
    "                                        batch_first=True)\n",
    "\n",
    "        self.rnn_layer2 = torch.nn.RNN(input_size=hidden_dim,\n",
    "                                        hidden_size=hidden_dim,\n",
    "                                        batch_first=True)\n",
    "\n",
    "        self.rnn_layer3 = torch.nn.RNN(input_size=hidden_dim,\n",
    "                                        hidden_size=hidden_dim,\n",
    "                                        batch_first=True)\n",
    "\n",
    "        # output層\n",
    "        self.linear = torch.nn.Sequential(torch.nn.Linear(in_features=hidden_dim,\n",
    "                                                          out_features=hidden_dim),\n",
    "                                          torch.nn.ReLU(),\n",
    "                                          torch.nn.Linear(in_features=hidden_dim,\n",
    "                                                          out_features=vocab_size))\n",
    "\n",
    "    def forward(self, batch_x, batch_x_lens):\n",
    "        return self.encoder(batch_x, batch_x_lens)\n",
    "\n",
    "    def encoder(self, batch_x, batch_x_lens):\n",
    "        batch_x = self.embedding(batch_x)\n",
    "        batch_x = torch.nn.utils.rnn.pack_padded_sequence(batch_x,\n",
    "                                                          batch_x_lens,\n",
    "                                                          batch_first=True,\n",
    "                                                          enforce_sorted=False)\n",
    "\n",
    "        batch_x, _ = self.rnn_layer1(batch_x)\n",
    "        batch_x, _ = self.rnn_layer2(batch_x)\n",
    "        batch_x, _ = self.rnn_layer3(batch_x)\n",
    "\n",
    "        batch_x, _ = torch.nn.utils.rnn.pad_packed_sequence(batch_x,\n",
    "                                                            batch_first=True)\n",
    "\n",
    "        batch_x = self.linear(batch_x)\n",
    "\n",
    "        return batch_x\n",
    "\n",
    "    def generator(self, start_char, max_len=200):\n",
    "\n",
    "        char_list = [char_to_id[start_char]]\n",
    "        next_char = None\n",
    "        # 生成的長度沒達到max_len就一直生\n",
    "        while len(char_list) < max_len:\n",
    "            x = torch.LongTensor(char_list).unsqueeze(0)\n",
    "            x = self.embedding(x)\n",
    "            _, (ht, _) = self.rnn_layer1(x)\n",
    "            _, (ht, _) = self.rnn_layer2(ht)\n",
    "            y = self.linear(ht)\n",
    "\n",
    "            next_char = np.argmax(y.numpy())\n",
    "\n",
    "            # 如果看到新的token是<eos>就說明生成結束了，就停下\n",
    "            if next_char == char_to_id['<eos>']:\n",
    "                break\n",
    "\n",
    "            char_list.append(next_char)\n",
    "\n",
    "        return [id_to_char[ch_id] for ch_id in char_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "J7J_jVDussdN",
    "outputId": "f11db567-5113-4d02-8774-00198b572a78"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(100)\n",
    "torch.cuda.manual_seed(100)\n",
    "\n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CharRNN(vocab_size, embed_dim, hidden_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "Z2XPQ9W4ssdN"
   },
   "outputs": [],
   "source": [
    "criterion = torch.nn.CrossEntropyLoss(reduction='mean', ignore_index=char_to_id['<pad>'])\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BXgU5hnpssdN"
   },
   "source": [
    "# 訓練\n",
    "1. 最外層的 `for` 迴圈控制 `epoch`\n",
    "    1. 內層的 `for` 迴圈透過 `data_loader` 取得 batch\n",
    "    2. 丟給 `model` 進行訓練\n",
    "    3. 預測結果 `batch_pred_y` 跟真正的答案 `batch_y` 進行 Cross Entropy 得到誤差 `loss`\n",
    "    4. 使用 `loss.backward` 自動計算梯度\n",
    "    5. 使用 `torch.nn.utils.clip_grad_value_` 將梯度限制在 `-grad_clip` &lt; &lt; `grad_clip` 之間\n",
    "    6. 使用 `optimizer.step()` 進行更新（back propagation）\n",
    "2. 每 `1000` 個 batch 就輸出一次當前的 loss 觀察是否有收斂的趨勢"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Model 1 Train\n",
    "batch_size = 64, lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jwAaDbGMssdN",
    "outputId": "26f1d09e-0e3b-4354-846d-0898b33e857e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training epoch 1: 100%|███████████████████████████| 12563/12563 [03:22<00:00, 62.03it/s, loss=0.741]\n",
      "Validation epoch 1: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 168.97it/s, loss=0.663]\n",
      "Training epoch 2: 100%|███████████████████████████| 12563/12563 [03:24<00:00, 61.30it/s, loss=0.659]\n",
      "Validation epoch 2: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 167.68it/s, loss=0.724]\n",
      "Training epoch 3: 100%|███████████████████████████| 12563/12563 [03:22<00:00, 62.03it/s, loss=0.675]\n",
      "Validation epoch 3: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 165.97it/s, loss=0.659]\n",
      "Training epoch 4: 100%|███████████████████████████| 12563/12563 [03:48<00:00, 55.03it/s, loss=0.609]\n",
      "Validation epoch 4: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 168.51it/s, loss=0.704]\n",
      "Training epoch 5: 100%|███████████████████████████| 12563/12563 [03:19<00:00, 63.08it/s, loss=0.668]\n",
      "Validation epoch 5: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 168.66it/s, loss=0.578]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "model = model.to(device)\n",
    "model.train()\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    i = 0\n",
    "    j = 0\n",
    "\n",
    "    process_bar = tqdm(data_loader_train_64, desc=f\"Training epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in process_bar:\n",
    "        optimizer.zero_grad()\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n",
    "        optimizer.step()\n",
    "        i+=1\n",
    "        if i%10==0:\n",
    "            process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            process_bar.update(1)\n",
    "\n",
    "    validation_process_bar = tqdm(data_loader_val_64, desc=f\"Validation epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in validation_process_bar:\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        j+=1\n",
    "        if j%10==0:\n",
    "            validation_process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            validation_process_bar.update(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AO7G73UIKd8-",
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Model 1 Test\n",
    "batch_size = 64, lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "HQseR0aOKd9P"
   },
   "outputs": [],
   "source": [
    "model.to(device)\n",
    "model.eval()\n",
    "\n",
    "x = []\n",
    "y = []\n",
    "pred_y = []\n",
    "\n",
    "j = 0\n",
    "\n",
    "batch_x, batch_y, batch_x_lens, batch_y_lens = next(iter(data_loader_test_64))\n",
    "batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "batch_y = batch_y.view(-1).to(device)\n",
    "\n",
    "\n",
    "x = batch_x\n",
    "y = batch_y.detach().cpu()\n",
    "pred_y = batch_pred_y.detach().cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "id": "QN1CuAU9F_vU"
   },
   "outputs": [],
   "source": [
    "def to_char(x):\n",
    "  return '' if x==char_to_id['<pad>'] else id_to_char[x]\n",
    "\n",
    "tc = np.vectorize(to_char)\n",
    "result_x = tc(x.numpy())\n",
    "result_y = np.reshape(tc(y.numpy()), [x.shape[0],-1])\n",
    "\n",
    "result_pred_y = np.empty_like(result_y)\n",
    "reshape_pred_y = np.reshape(pred_y.numpy(), [x.shape[0],x.shape[1],-1])\n",
    "for i in range(x.shape[0]):\n",
    "  for j in range(x.shape[1]):\n",
    "      result_pred_y[i,j] = tc(np.argmax(reshape_pred_y[i,j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_result(y, pred_y):\n",
    "    output_pred_y = pred_y\n",
    "    for i in range(y.shape[0]):\n",
    "        for j in range(y.shape[1]):\n",
    "            if y[i,j] == '':\n",
    "                output_pred_y[i,j] = ''\n",
    "    return y, output_pred_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "id": "9vUmY3NvGcGG",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input               output         answer         \n",
      "47+11-36=           23<eos>        22<eos>        wrong\n",
      "29-2+30=            55<eos>        57<eos>        wrong\n",
      "(19-15)+28=         32<eos>        32<eos>        correct\n",
      "7+(39-8)=           39<eos>        38<eos>        wrong\n",
      "14+(14-23)=         6<eos>         5<eos>         wrong\n",
      "(23-13)+23=         32<eos>        33<eos>        wrong\n",
      "(46+6)-39=          14<eos>        13<eos>        wrong\n",
      "11-19-17=           -23<eos>       -25<eos>       wrong\n",
      "(46-49)+13=         10<eos>        10<eos>        correct\n",
      "41-(13+40)=         -10<eos>       -12<eos>       wrong\n",
      "25+45+2=            74<eos>        72<eos>        wrong\n",
      "(6-37)+11=          -20<eos>       -20<eos>       correct\n",
      "(33+27)-32=         27<eos>        28<eos>        wrong\n",
      "(19+18)-29=         8<eos>         8<eos>         correct\n",
      "26+(0-30)=          -4<eos>        -4<eos>        correct\n",
      "...\n",
      "total: 64 samples\n",
      "accuracy = 0.31\n"
     ]
    }
   ],
   "source": [
    "print(\"{:<20}{:<15}{:<15}\".format(\"input\", \"output\", \"answer\"))\n",
    "\n",
    "correct = .0\n",
    "\n",
    "output_y, output_pred_y = get_result(result_y, result_pred_y)\n",
    "\n",
    "for i in range(result_x.shape[0]):\n",
    "    if i<15:\n",
    "        print(\"{:<20}{:<15}{:<15}{:<5}\".format(''.join(result_x[i]), \n",
    "                                             ''.join(output_pred_y[i]), \n",
    "                                             ''.join(output_y[i]), \n",
    "                                             'correct' if (''.join(output_pred_y[i])==''.join(output_y[i])) else 'wrong'\n",
    "                                            )\n",
    "           )\n",
    "    if ''.join(output_pred_y[i])==''.join(output_y[i]):\n",
    "        correct = correct + 1.\n",
    "        \n",
    "print(\"...\\ntotal: {:} samples\".format(result_x.shape[0]))\n",
    "print(\"accuracy = {:.2f}\".format(correct / result_x.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Model 2 Train\n",
    "batch_size = 128, lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training epoch 1: 100%|█████████████████████████████| 6282/6282 [02:18<00:00, 45.20it/s, loss=2.901]\n",
      "Validation epoch 1: 100%|████████████████████████████| 786/786 [00:07<00:00, 107.32it/s, loss=2.899]\n",
      "Training epoch 2: 100%|█████████████████████████████| 6282/6282 [01:55<00:00, 54.20it/s, loss=2.899]\n",
      "Validation epoch 2: 100%|████████████████████████████| 786/786 [00:07<00:00, 106.00it/s, loss=2.902]\n",
      "Training epoch 3: 100%|█████████████████████████████| 6282/6282 [02:15<00:00, 46.36it/s, loss=2.900]\n",
      "Validation epoch 3: 100%|████████████████████████████| 786/786 [00:07<00:00, 104.11it/s, loss=2.899]\n",
      "Training epoch 4: 100%|█████████████████████████████| 6282/6282 [02:26<00:00, 42.75it/s, loss=2.895]\n",
      "Validation epoch 4: 100%|████████████████████████████| 786/786 [00:07<00:00, 107.04it/s, loss=2.899]\n",
      "Training epoch 5: 100%|█████████████████████████████| 6282/6282 [01:47<00:00, 58.44it/s, loss=2.896]\n",
      "Validation epoch 5: 100%|█████████████████████████████| 786/786 [00:08<00:00, 92.78it/s, loss=2.897]\n"
     ]
    }
   ],
   "source": [
    "model = CharRNN(vocab_size, embed_dim, hidden_dim)\n",
    "model = model.to(device)\n",
    "model.train()\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    i = 0\n",
    "    j = 0\n",
    "\n",
    "    process_bar = tqdm(data_loader_train_128, desc=f\"Training epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in process_bar:\n",
    "        optimizer.zero_grad()\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n",
    "        optimizer.step()\n",
    "        i+=1\n",
    "        if i%10==0:\n",
    "            process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            process_bar.update(1)\n",
    "\n",
    "    validation_process_bar = tqdm(data_loader_val_128, desc=f\"Validation epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in validation_process_bar:\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        j+=1\n",
    "        if j%10==0:\n",
    "            validation_process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            validation_process_bar.update(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Model 2 Test\n",
    "batch_size = 128, lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "id": "HQseR0aOKd9P"
   },
   "outputs": [],
   "source": [
    "model.to(device)\n",
    "model.eval()\n",
    "\n",
    "x = []\n",
    "y = []\n",
    "pred_y = []\n",
    "\n",
    "j = 0\n",
    "\n",
    "batch_x, batch_y, batch_x_lens, batch_y_lens = next(iter(data_loader_test_128))\n",
    "batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "batch_y = batch_y.view(-1).to(device)\n",
    "\n",
    "\n",
    "x = batch_x\n",
    "y = batch_y.detach().cpu()\n",
    "pred_y = batch_pred_y.detach().cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "id": "QN1CuAU9F_vU"
   },
   "outputs": [],
   "source": [
    "def to_char(x):\n",
    "  return '' if x==char_to_id['<pad>'] else id_to_char[x]\n",
    "\n",
    "tc = np.vectorize(to_char)\n",
    "result_x = tc(x.numpy())\n",
    "result_y = np.reshape(tc(y.numpy()), [x.shape[0],-1])\n",
    "\n",
    "result_pred_y = np.empty_like(result_y)\n",
    "reshape_pred_y = np.reshape(pred_y.numpy(), [x.shape[0],x.shape[1],-1])\n",
    "for i in range(x.shape[0]):\n",
    "  for j in range(x.shape[1]):\n",
    "      result_pred_y[i,j] = tc(np.argmax(reshape_pred_y[i,j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "id": "9vUmY3NvGcGG"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input               output         answer         \n",
      "(14-1)+16=          (99            29<eos>        wrong\n",
      "(8+7)-39=           99((           -24<eos>       wrong\n",
      "30+29+43=           (4((           102<eos>       wrong\n",
      "5+(46-15)=          944            36<eos>        wrong\n",
      "26+18-42=           *(             2<eos>         wrong\n",
      "37+(47-8)=          944            76<eos>        wrong\n",
      "12-15+25=           *(9            22<eos>        wrong\n",
      "(29-0)+17=          *44            46<eos>        wrong\n",
      "22+(10-38)=         (49            -6<eos>        wrong\n",
      "(15+41)-18=         *9(            38<eos>        wrong\n",
      "10+33+15=           4((            58<eos>        wrong\n",
      "(37-12)+36=         949            61<eos>        wrong\n",
      "12-(3+24)=          944(           -15<eos>       wrong\n",
      "17-(10+21)=         (4((           -14<eos>       wrong\n",
      "49-43-17=           *4((           -11<eos>       wrong\n",
      "...\n",
      "total: 128 samples\n",
      "accuracy = 0.00\n"
     ]
    }
   ],
   "source": [
    "print(\"{:<20}{:<15}{:<15}\".format(\"input\", \"output\", \"answer\"))\n",
    "\n",
    "correct = .0\n",
    "\n",
    "output_y, output_pred_y = get_result(result_y, result_pred_y)\n",
    "\n",
    "for i in range(result_x.shape[0]):\n",
    "    if i<15:\n",
    "        print(\"{:<20}{:<15}{:<15}{:<5}\".format(''.join(result_x[i]), \n",
    "                                             ''.join(output_pred_y[i]), \n",
    "                                             ''.join(output_y[i]), \n",
    "                                             'correct' if (''.join(output_pred_y[i])==''.join(output_y[i])) else 'wrong'\n",
    "                                            )\n",
    "           )\n",
    "    if ''.join(output_pred_y[i])==''.join(output_y[i]):\n",
    "        correct = correct + 1.\n",
    "        \n",
    "print(\"...\\ntotal: {:} samples\".format(result_x.shape[0]))\n",
    "print(\"accuracy = {:.2f}\".format(correct / result_x.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Model 3 Train\n",
    "batch_size = 64, lr = 0.002"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.002"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training epoch 1: 100%|███████████████████████████| 12563/12563 [03:42<00:00, 56.59it/s, loss=2.895]\n",
      "Validation epoch 1: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 165.13it/s, loss=2.897]\n",
      "Training epoch 2: 100%|███████████████████████████| 12563/12563 [02:38<00:00, 79.19it/s, loss=2.888]\n",
      "Validation epoch 2: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 169.23it/s, loss=2.889]\n",
      "Training epoch 3: 100%|███████████████████████████| 12563/12563 [03:09<00:00, 66.29it/s, loss=2.889]\n",
      "Validation epoch 3: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 167.63it/s, loss=2.893]\n",
      "Training epoch 4: 100%|███████████████████████████| 12563/12563 [02:47<00:00, 74.96it/s, loss=2.887]\n",
      "Validation epoch 4: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 167.52it/s, loss=2.895]\n",
      "Training epoch 5: 100%|███████████████████████████| 12563/12563 [02:31<00:00, 83.01it/s, loss=2.891]\n",
      "Validation epoch 5: 100%|██████████████████████████| 1571/1571 [00:09<00:00, 164.32it/s, loss=2.887]\n"
     ]
    }
   ],
   "source": [
    "model = CharRNN(vocab_size, embed_dim, hidden_dim)\n",
    "model = model.to(device)\n",
    "model.train()\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    i = 0\n",
    "    j = 0\n",
    "\n",
    "    process_bar = tqdm(data_loader_train_64, desc=f\"Training epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in process_bar:\n",
    "        optimizer.zero_grad()\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n",
    "        optimizer.step()\n",
    "        i+=1\n",
    "        if i%10==0:\n",
    "            # process_bar.set_postfix(loss=loss.item())\n",
    "            process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            process_bar.update(1)\n",
    "\n",
    "    validation_process_bar = tqdm(data_loader_val_64, desc=f\"Validation epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in validation_process_bar:\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        j+=1\n",
    "        if j%10==0:\n",
    "            validation_process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            validation_process_bar.update(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Model 3 Test\n",
    "batch_size = 64, lr = 0.025"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.to(device)\n",
    "model.eval()\n",
    "\n",
    "x = []\n",
    "y = []\n",
    "pred_y = []\n",
    "\n",
    "j = 0\n",
    "\n",
    "batch_x, batch_y, batch_x_lens, batch_y_lens = next(iter(data_loader_test_128))\n",
    "batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "batch_y = batch_y.view(-1).to(device)\n",
    "\n",
    "\n",
    "x = batch_x\n",
    "y = batch_y.detach().cpu()\n",
    "pred_y = batch_pred_y.detach().cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "tc = np.vectorize(to_char)\n",
    "result_x = tc(x.numpy())\n",
    "result_y = np.reshape(tc(y.numpy()), [x.shape[0],-1])\n",
    "\n",
    "result_pred_y = np.empty_like(result_y)\n",
    "reshape_pred_y = np.reshape(pred_y.numpy(), [x.shape[0],x.shape[1],-1])\n",
    "for i in range(x.shape[0]):\n",
    "  for j in range(x.shape[1]):\n",
    "      result_pred_y[i,j] = tc(np.argmax(reshape_pred_y[i,j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input               output         answer         \n",
      "45+(29-38)=         (<eos>(        36<eos>        wrong\n",
      "46-3+15=            <eos>(         58<eos>        wrong\n",
      "47+(4-28)=          (+(            23<eos>        wrong\n",
      "34-(40+12)=         (<eos>(6       -18<eos>       wrong\n",
      "(22-46)+41=         +6             17<eos>        wrong\n",
      "3+10-44=            6<eos>(6       -31<eos>       wrong\n",
      "23-49+14=           <eos>(6        -12<eos>       wrong\n",
      "(49-27)+48=         (<eos>(        70<eos>        wrong\n",
      "28-(49+32)=         (<eos>(6       -53<eos>       wrong\n",
      "(47-27)+15=         <eos>(         35<eos>        wrong\n",
      "22-(44+24)=         (6(6           -46<eos>       wrong\n",
      "45+16-0=            6<eos>6        61<eos>        wrong\n",
      "17+1-42=            +<eos>66       -24<eos>       wrong\n",
      "18-15-45=           <eos>(6        -42<eos>       wrong\n",
      "35-49-28=           6<eos><eos>6   -42<eos>       wrong\n",
      "...\n",
      "total: 128 samples\n",
      "accuracy = 0.01\n"
     ]
    }
   ],
   "source": [
    "print(\"{:<20}{:<15}{:<15}\".format(\"input\", \"output\", \"answer\"))\n",
    "\n",
    "correct = .0\n",
    "\n",
    "output_y, output_pred_y = get_result(result_y, result_pred_y)\n",
    "\n",
    "for i in range(result_x.shape[0]):\n",
    "    if i<15:\n",
    "        print(\"{:<20}{:<15}{:<15}{:<5}\".format(''.join(result_x[i]), \n",
    "                                             ''.join(output_pred_y[i]), \n",
    "                                             ''.join(output_y[i]), \n",
    "                                             'correct' if (''.join(output_pred_y[i])==''.join(output_y[i])) else 'wrong'\n",
    "                                            )\n",
    "           )\n",
    "    if ''.join(output_pred_y[i])==''.join(output_y[i]):\n",
    "        correct = correct + 1.\n",
    "        \n",
    "print(\"...\\ntotal: {:} samples\".format(result_x.shape[0]))\n",
    "print(\"accuracy = {:.2f}\".format(correct / result_x.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Dataset 2 with Model 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df2 = df[~df['src'].str.contains(\"\\*\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "upper = math.ceil(new_df2['src'].str.len().mean() + 2 * new_df2['src'].str.len().std())\n",
    "lower = math.floor(new_df2['src'].str.len().mean() - 2 * new_df2['src'].str.len().std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_17844/2375016477.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  new_df2['len'] = new_df2['src'].str.len()\n"
     ]
    }
   ],
   "source": [
    "new_df2['len'] = new_df2['src'].str.len()\n",
    "new_df2 = new_df2[(new_df2['len'] >= lower) & (new_df2['len'] <= upper)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "xZDW6Zmz2RcC"
   },
   "outputs": [],
   "source": [
    "# 把資料轉成id\n",
    "new_df2['src'] = new_df2['src'].apply(lambda text: [char_to_id[char] for char in text])\n",
    "new_df2['tgt'] = new_df2['tgt'].apply(lambda num: [char_to_id[char] for char in str(num)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把資料分成train, val, test = 0.8 : 0.1 : 0.1\n",
    "train_data2, val_data2 = train_test_split(new_df2, test_size=0.2, random_state=250)\n",
    "val_data2, test_data2 = train_test_split(val_data2, test_size=0.5, random_state=250)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "Rd4bWLP1ssdM"
   },
   "outputs": [],
   "source": [
    "dataset2_train = Dataset(train_data2)\n",
    "dataset2_val = Dataset(val_data2)\n",
    "dataset2_test = Dataset(test_data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "DYp9q3e5ssdM"
   },
   "outputs": [],
   "source": [
    "data_loader_train_64 = torch.utils.data.DataLoader(dataset2_train,\n",
    "                                                    batch_size=batch_size,\n",
    "                                                    shuffle=True,\n",
    "                                                    collate_fn=collate_fn)\n",
    "\n",
    "data_loader_val_64 = torch.utils.data.DataLoader(dataset2_val,\n",
    "                                                    batch_size=batch_size,\n",
    "                                                    shuffle=True,\n",
    "                                                    collate_fn=collate_fn)\n",
    "\n",
    "data_loader_test_64 = torch.utils.data.DataLoader(dataset2_test,\n",
    "                                                    batch_size=batch_size,\n",
    "                                                    shuffle=True,\n",
    "                                                    collate_fn=collate_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jwAaDbGMssdN",
    "outputId": "26f1d09e-0e3b-4354-846d-0898b33e857e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training epoch 1: 100%|███████████████████████████| 12540/12540 [03:03<00:00, 68.39it/s, loss=0.641]\n",
      "Validation epoch 1: 100%|██████████████████████████| 1568/1568 [00:09<00:00, 168.74it/s, loss=0.649]\n",
      "Training epoch 2: 100%|███████████████████████████| 12540/12540 [02:36<00:00, 80.06it/s, loss=0.739]\n",
      "Validation epoch 2: 100%|██████████████████████████| 1568/1568 [00:09<00:00, 168.68it/s, loss=0.681]\n",
      "Training epoch 3: 100%|███████████████████████████| 12540/12540 [02:48<00:00, 74.22it/s, loss=0.726]\n",
      "Validation epoch 3: 100%|██████████████████████████| 1568/1568 [00:10<00:00, 154.44it/s, loss=0.649]\n",
      "Training epoch 4: 100%|███████████████████████████| 12540/12540 [02:49<00:00, 73.91it/s, loss=0.597]\n",
      "Validation epoch 4: 100%|██████████████████████████| 1568/1568 [00:10<00:00, 144.24it/s, loss=0.690]\n",
      "Training epoch 5: 100%|███████████████████████████| 12540/12540 [03:19<00:00, 62.90it/s, loss=0.657]\n",
      "Validation epoch 5: 100%|██████████████████████████| 1568/1568 [00:09<00:00, 168.27it/s, loss=0.673]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "model = model.to(device)\n",
    "model.train()\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    i = 0\n",
    "    j = 0\n",
    "\n",
    "    process_bar = tqdm(data_loader_train_64, desc=f\"Training epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in process_bar:\n",
    "        optimizer.zero_grad()\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n",
    "        optimizer.step()\n",
    "        i+=1\n",
    "        if i%10==0:\n",
    "            process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            process_bar.update(1)\n",
    "\n",
    "    validation_process_bar = tqdm(data_loader_val_64, desc=f\"Validation epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in validation_process_bar:\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        j+=1\n",
    "        if j%10==0:\n",
    "            validation_process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            validation_process_bar.update(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AO7G73UIKd8-"
   },
   "source": [
    "## Test\n",
    "batch_size = 64, lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "HQseR0aOKd9P"
   },
   "outputs": [],
   "source": [
    "model.to(device)\n",
    "model.eval()\n",
    "\n",
    "x = []\n",
    "y = []\n",
    "pred_y = []\n",
    "\n",
    "j = 0\n",
    "\n",
    "batch_x, batch_y, batch_x_lens, batch_y_lens = next(iter(data_loader_test_64))\n",
    "batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "batch_y = batch_y.view(-1).to(device)\n",
    "\n",
    "\n",
    "x = batch_x\n",
    "y = batch_y.detach().cpu()\n",
    "pred_y = batch_pred_y.detach().cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "QN1CuAU9F_vU"
   },
   "outputs": [],
   "source": [
    "def to_char(x):\n",
    "  return '' if x==char_to_id['<pad>'] else id_to_char[x]\n",
    "\n",
    "tc = np.vectorize(to_char)\n",
    "result_x = tc(x.numpy())\n",
    "result_y = np.reshape(tc(y.numpy()), [x.shape[0],-1])\n",
    "\n",
    "result_pred_y = np.empty_like(result_y)\n",
    "reshape_pred_y = np.reshape(pred_y.numpy(), [x.shape[0],x.shape[1],-1])\n",
    "for i in range(x.shape[0]):\n",
    "  for j in range(x.shape[1]):\n",
    "      result_pred_y[i,j] = tc(np.argmax(reshape_pred_y[i,j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_result(y, pred_y):\n",
    "    output_pred_y = pred_y\n",
    "    for i in range(y.shape[0]):\n",
    "        for j in range(y.shape[1]):\n",
    "            if y[i,j] == '':\n",
    "                output_pred_y[i,j] = ''\n",
    "    return y, output_pred_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "9vUmY3NvGcGG",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input               output         answer         \n",
      "23+(40-48)=         16<eos>        15<eos>        wrong\n",
      "44-2-46=            -4<eos>        -4<eos>        correct\n",
      "14+44+10=           67<eos>        68<eos>        wrong\n",
      "6-4+34=             36<eos>        36<eos>        correct\n",
      "(3-10)+12=          7<eos>         5<eos>         wrong\n",
      "36-15+6=            27<eos>        27<eos>        correct\n",
      "(21+2)-45=          -22<eos>       -22<eos>       correct\n",
      "20-6-33=            -18<eos>       -19<eos>       wrong\n",
      "(38+34)-30=         41<eos>        42<eos>        wrong\n",
      "4-28-45=            -60<eos>       -69<eos>       wrong\n",
      "7-(32+10)=          -40<eos>       -35<eos>       wrong\n",
      "35+(49-5)=          73<eos>        79<eos>        wrong\n",
      "6+(38-38)=          6<eos>         6<eos>         correct\n",
      "46+(1-28)=          10<eos>        19<eos>        wrong\n",
      "(9-13)+49=          46<eos>        45<eos>        wrong\n",
      "...\n",
      "total: 64 samples\n",
      "accuracy = 0.19\n"
     ]
    }
   ],
   "source": [
    "print(\"{:<20}{:<15}{:<15}\".format(\"input\", \"output\", \"answer\"))\n",
    "\n",
    "correct = .0\n",
    "\n",
    "output_y, output_pred_y = get_result(result_y, result_pred_y)\n",
    "\n",
    "for i in range(result_x.shape[0]):\n",
    "    if i<15:\n",
    "        print(\"{:<20}{:<15}{:<15}{:<5}\".format(''.join(result_x[i]), \n",
    "                                             ''.join(output_pred_y[i]), \n",
    "                                             ''.join(output_y[i]), \n",
    "                                             'correct' if (''.join(output_pred_y[i])==''.join(output_y[i])) else 'wrong'\n",
    "                                            )\n",
    "           )\n",
    "    if ''.join(output_pred_y[i])==''.join(output_y[i]):\n",
    "        correct = correct + 1.\n",
    "        \n",
    "print(\"...\\ntotal: {:} samples\".format(result_x.shape[0]))\n",
    "print(\"accuracy = {:.2f}\".format(correct / result_x.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Origianl Dataset with Model 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把資料轉成id\n",
    "df_original = df\n",
    "df_original['src'] = df_original['src'].apply(lambda text: [char_to_id[char] for char in text])\n",
    "df_original['tgt'] = df_original['tgt'].apply(lambda num: [char_to_id[char] for char in str(num)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把資料分成train, val, test = 0.8 : 0.1 : 0.1\n",
    "train_data_original, val_data_original = train_test_split(df_original, test_size=0.2, random_state=250)\n",
    "val_data_original, test_data_original = train_test_split(val_data_original, test_size=0.5, random_state=250)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "Rd4bWLP1ssdM"
   },
   "outputs": [],
   "source": [
    "dataset_original_train = Dataset(train_data_original)\n",
    "dataset_original_val = Dataset(val_data_original)\n",
    "dataset_original_test = Dataset(test_data_original)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "DYp9q3e5ssdM"
   },
   "outputs": [],
   "source": [
    "data_loader_train_orignal = torch.utils.data.DataLoader(dataset_original_train,\n",
    "                                                    batch_size=batch_size,\n",
    "                                                    shuffle=True,\n",
    "                                                    collate_fn=collate_fn)\n",
    "\n",
    "data_loader_val_orignal = torch.utils.data.DataLoader(dataset_original_val,\n",
    "                                                    batch_size=batch_size,\n",
    "                                                    shuffle=True,\n",
    "                                                    collate_fn=collate_fn)\n",
    "\n",
    "data_loader_test_orignal = torch.utils.data.DataLoader(dataset_original_test,\n",
    "                                                    batch_size=batch_size,\n",
    "                                                    shuffle=True,\n",
    "                                                    collate_fn=collate_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jwAaDbGMssdN",
    "outputId": "26f1d09e-0e3b-4354-846d-0898b33e857e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training epoch 1: 100%|███████████████████████████| 32907/32907 [13:14<00:00, 41.40it/s, loss=1.086]\n",
      "Validation epoch 1: 100%|███████████████████████████| 4114/4114 [00:47<00:00, 86.95it/s, loss=1.104]\n",
      "Training epoch 2: 100%|███████████████████████████| 32907/32907 [12:48<00:00, 42.79it/s, loss=1.070]\n",
      "Validation epoch 2: 100%|███████████████████████████| 4114/4114 [00:43<00:00, 94.21it/s, loss=1.139]\n",
      "Training epoch 3: 100%|███████████████████████████| 32907/32907 [12:19<00:00, 44.50it/s, loss=1.265]\n",
      "Validation epoch 3: 100%|███████████████████████████| 4114/4114 [00:43<00:00, 95.42it/s, loss=1.046]\n",
      "Training epoch 4: 100%|███████████████████████████| 32907/32907 [12:09<00:00, 45.10it/s, loss=1.078]\n",
      "Validation epoch 4: 100%|███████████████████████████| 4114/4114 [00:44<00:00, 93.23it/s, loss=1.141]\n",
      "Training epoch 5: 100%|███████████████████████████| 32907/32907 [12:30<00:00, 43.87it/s, loss=1.145]\n",
      "Validation epoch 5: 100%|███████████████████████████| 4114/4114 [00:48<00:00, 84.24it/s, loss=1.113]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "model = model.to(device)\n",
    "model.train()\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    i = 0\n",
    "    j = 0\n",
    "\n",
    "    process_bar = tqdm(data_loader_train_orignal, desc=f\"Training epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in process_bar:\n",
    "        optimizer.zero_grad()\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n",
    "        optimizer.step()\n",
    "        i+=1\n",
    "        if i%10==0:\n",
    "            process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            process_bar.update(1)\n",
    "\n",
    "    validation_process_bar = tqdm(data_loader_val_orignal, desc=f\"Validation epoch {epoch}\", ncols=100)\n",
    "    for batch_x, batch_y, batch_x_lens, batch_y_lens in validation_process_bar:\n",
    "        batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "        batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "        batch_y = batch_y.view(-1).to(device)\n",
    "        loss = criterion(batch_pred_y, batch_y)\n",
    "        j+=1\n",
    "        if j%10==0:\n",
    "            validation_process_bar.set_postfix(loss=\"{:.3f}\".format(loss.item()))\n",
    "            validation_process_bar.update(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AO7G73UIKd8-"
   },
   "source": [
    "## Test\n",
    "batch_size = 64, lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "HQseR0aOKd9P"
   },
   "outputs": [],
   "source": [
    "model.to(device)\n",
    "model.eval()\n",
    "\n",
    "x = []\n",
    "y = []\n",
    "pred_y = []\n",
    "\n",
    "j = 0\n",
    "\n",
    "batch_x, batch_y, batch_x_lens, batch_y_lens = next(iter(data_loader_test_orignal))\n",
    "batch_pred_y = model(batch_x.to(device), batch_x_lens)\n",
    "batch_pred_y = batch_pred_y.view(-1, vocab_size)\n",
    "batch_y = batch_y.view(-1).to(device)\n",
    "\n",
    "\n",
    "x = batch_x\n",
    "y = batch_y.detach().cpu()\n",
    "pred_y = batch_pred_y.detach().cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "QN1CuAU9F_vU"
   },
   "outputs": [],
   "source": [
    "def to_char(x):\n",
    "  return '' if x==char_to_id['<pad>'] else id_to_char[x]\n",
    "\n",
    "tc = np.vectorize(to_char)\n",
    "result_x = tc(x.numpy())\n",
    "result_y = np.reshape(tc(y.numpy()), [x.shape[0],-1])\n",
    "\n",
    "result_pred_y = np.empty_like(result_y)\n",
    "reshape_pred_y = np.reshape(pred_y.numpy(), [x.shape[0],x.shape[1],-1])\n",
    "for i in range(x.shape[0]):\n",
    "  for j in range(x.shape[1]):\n",
    "      result_pred_y[i,j] = tc(np.argmax(reshape_pred_y[i,j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_result(y, pred_y):\n",
    "    output_pred_y = pred_y\n",
    "    for i in range(y.shape[0]):\n",
    "        for j in range(y.shape[1]):\n",
    "            if y[i,j] == '':\n",
    "                output_pred_y[i,j] = ''\n",
    "    return y, output_pred_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "9vUmY3NvGcGG",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input               output         answer         \n",
      "26*14-45=           301<eos>       319<eos>       wrong\n",
      "46+(38*26)=         900<eos><eos>  1034<eos>      wrong\n",
      "(39+8)-36=          12<eos>        11<eos>        wrong\n",
      "(31+16)*14=         778<eos>       658<eos>       wrong\n",
      "(49-27)*9=          177<eos>       198<eos>       wrong\n",
      "19*16*3=            100<eos>       912<eos>       wrong\n",
      "(19*41)+10=         700<eos>       789<eos>       wrong\n",
      "10*46-31=           401<eos>       429<eos>       wrong\n",
      "40-25*15=           -200<eos>      -335<eos>      wrong\n",
      "27+(45-8)=          60<eos>        64<eos>        wrong\n",
      "(32+24)*9=          570<eos>       504<eos>       wrong\n",
      "23*47-47=           107<eos><eos>  1034<eos>      wrong\n",
      "32-(30+34)=         -32<eos>       -32<eos>       correct\n",
      "32+(46*36)=         1672<eos>      1688<eos>      wrong\n",
      "49*5*41=            1070<eos><eos> 10045<eos>     wrong\n",
      "...\n",
      "total: 64 samples\n",
      "accuracy = 0.06\n"
     ]
    }
   ],
   "source": [
    "print(\"{:<20}{:<15}{:<15}\".format(\"input\", \"output\", \"answer\"))\n",
    "\n",
    "correct = .0\n",
    "\n",
    "output_y, output_pred_y = get_result(result_y, result_pred_y)\n",
    "\n",
    "for i in range(result_x.shape[0]):\n",
    "    if i<15:\n",
    "        print(\"{:<20}{:<15}{:<15}{:<5}\".format(''.join(result_x[i]), \n",
    "                                             ''.join(output_pred_y[i]), \n",
    "                                             ''.join(output_y[i]), \n",
    "                                             'correct' if (''.join(output_pred_y[i])==''.join(output_y[i])) else 'wrong'\n",
    "                                            )\n",
    "           )\n",
    "    if ''.join(output_pred_y[i])==''.join(output_y[i]):\n",
    "        correct = correct + 1.\n",
    "        \n",
    "print(\"...\\ntotal: {:} samples\".format(result_x.shape[0]))\n",
    "print(\"accuracy = {:.2f}\".format(correct / result_x.shape[0]))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}