torchtext.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Creating a shortcut link for 'en' didn't work (maybe you don't have admin\n",
    "permissions?), but you can still load the model via its full package name: nlp =\n",
    "spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      ".data\\ag_news_csv.tar.gz: 11.8MB [00:03, 3.55MB/s]\n",
      "120000lines [00:07, 16133.16lines/s]\n",
      "120000lines [00:13, 8902.11lines/s]\n",
      "7600lines [00:00, 9170.10lines/s]\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchtext\n",
    "from torchtext.datasets import text_classification\n",
    "NGRAMS = 2\n",
    "import os\n",
    "if not os.path.isdir('./.data'):\n",
    "    os.mkdir('./.data')\n",
    "train_dataset, test_dataset = text_classification.DATASETS['AG_NEWS'](root='./.data', ngrams=NGRAMS, vocab=None)\n",
    "BATCH_SIZE = 16\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "class TextSentiment(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_dim, num_class):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.EmbeddingBag(vocab_size, embed_dim, sparse=True)\n",
    "        self.fc = nn.Linear(embed_dim, num_class)\n",
    "        self.init_weights()\n",
    "\n",
    "    def init_weights(self):\n",
    "        initrange = 0.5\n",
    "        self.embedding.weight.data.uniform_(-initrange, initrange)\n",
    "        self.fc.weight.data.uniform_(-initrange, initrange)\n",
    "        self.fc.bias.data.zero_()\n",
    "\n",
    "    def forward(self, text, offsets):\n",
    "        embedded = self.embedding(text, offsets)\n",
    "        return self.fc(embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "VOCAB_SIZE = len(train_dataset.get_vocab())\n",
    "EMBED_DIM = 32\n",
    "NUN_CLASS = len(train_dataset.get_labels())\n",
    "model = TextSentiment(VOCAB_SIZE, EMBED_DIM, NUN_CLASS).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_batch(batch):\n",
    "    label = torch.tensor([entry[0] for entry in batch])\n",
    "    text = [entry[1] for entry in batch]\n",
    "    offsets = [0] + [len(entry) for entry in text]\n",
    "    # torch.Tensor.cumsum returns the cumulative sum\n",
    "    # of elements in the dimension dim.\n",
    "    # torch.Tensor([1.0, 2.0, 3.0]).cumsum(dim=0)\n",
    "\n",
    "    offsets = torch.tensor(offsets[:-1]).cumsum(dim=0)\n",
    "    text = torch.cat(text)\n",
    "    return text, offsets, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "def train_func(sub_train_):\n",
    "\n",
    "    # Train the model\n",
    "    train_loss = 0\n",
    "    train_acc = 0\n",
    "    data = DataLoader(sub_train_, batch_size=BATCH_SIZE, shuffle=True,\n",
    "                      collate_fn=generate_batch)\n",
    "    for i, (text, offsets, cls) in enumerate(data):\n",
    "        optimizer.zero_grad()\n",
    "        text, offsets, cls = text.to(device), offsets.to(device), cls.to(device)\n",
    "        output = model(text, offsets)\n",
    "        loss = criterion(output, cls)\n",
    "        train_loss += loss.item()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        train_acc += (output.argmax(1) == cls).sum().item()\n",
    "\n",
    "    # Adjust the learning rate\n",
    "    scheduler.step()\n",
    "\n",
    "    return train_loss / len(sub_train_), train_acc / len(sub_train_)\n",
    "\n",
    "def test(data_):\n",
    "    loss = 0\n",
    "    acc = 0\n",
    "    data = DataLoader(data_, batch_size=BATCH_SIZE, collate_fn=generate_batch)\n",
    "    for text, offsets, cls in data:\n",
    "        text, offsets, cls = text.to(device), offsets.to(device), cls.to(device)\n",
    "        with torch.no_grad():\n",
    "            output = model(text, offsets)\n",
    "            loss = criterion(output, cls)\n",
    "            loss += loss.item()\n",
    "            acc += (output.argmax(1) == cls).sum().item()\n",
    "\n",
    "    return loss / len(data_), acc / len(data_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1  | time in 0 minutes, 22 seconds\n",
      "\tLoss: 0.0261(train)\t|\tAcc: 84.8%(train)\n",
      "\tLoss: 0.0001(valid)\t|\tAcc: 90.7%(valid)\n",
      "Epoch: 2  | time in 0 minutes, 22 seconds\n",
      "\tLoss: 0.0119(train)\t|\tAcc: 93.7%(train)\n",
      "\tLoss: 0.0000(valid)\t|\tAcc: 91.6%(valid)\n",
      "Epoch: 3  | time in 0 minutes, 22 seconds\n",
      "\tLoss: 0.0069(train)\t|\tAcc: 96.4%(train)\n",
      "\tLoss: 0.0001(valid)\t|\tAcc: 90.9%(valid)\n",
      "Epoch: 4  | time in 0 minutes, 22 seconds\n",
      "\tLoss: 0.0039(train)\t|\tAcc: 98.1%(train)\n",
      "\tLoss: 0.0000(valid)\t|\tAcc: 91.7%(valid)\n",
      "Epoch: 5  | time in 0 minutes, 22 seconds\n",
      "\tLoss: 0.0023(train)\t|\tAcc: 99.0%(train)\n",
      "\tLoss: 0.0001(valid)\t|\tAcc: 91.7%(valid)\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from torch.utils.data.dataset import random_split\n",
    "N_EPOCHS = 5\n",
    "min_valid_loss = float('inf')\n",
    "\n",
    "criterion = torch.nn.CrossEntropyLoss().to(device)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=4.0)\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, gamma=0.9)\n",
    "\n",
    "train_len = int(len(train_dataset) * 0.95)\n",
    "sub_train_, sub_valid_ = \\\n",
    "    random_split(train_dataset, [train_len, len(train_dataset) - train_len])\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    train_loss, train_acc = train_func(sub_train_)\n",
    "    valid_loss, valid_acc = test(sub_valid_)\n",
    "\n",
    "    secs = int(time.time() - start_time)\n",
    "    mins = secs / 60\n",
    "    secs = secs % 60\n",
    "\n",
    "    print('Epoch: %d' %(epoch + 1), \" | time in %d minutes, %d seconds\" %(mins, secs))\n",
    "    print(f'\\tLoss: {train_loss:.4f}(train)\\t|\\tAcc: {train_acc * 100:.1f}%(train)')\n",
    "    print(f'\\tLoss: {valid_loss:.4f}(valid)\\t|\\tAcc: {valid_acc * 100:.1f}%(valid)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking the results of test dataset...\n",
      "\tLoss: 0.0002(test)\t|\tAcc: 90.1%(test)\n"
     ]
    }
   ],
   "source": [
    "print('Checking the results of test dataset...')\n",
    "test_loss, test_acc = test(test_dataset)\n",
    "print(f'\\tLoss: {test_loss:.4f}(test)\\t|\\tAcc: {test_acc * 100:.1f}%(test)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is a Sports news\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "from torchtext.data.utils import ngrams_iterator\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "\n",
    "ag_news_label = {1 : \"World\",\n",
    "                 2 : \"Sports\",\n",
    "                 3 : \"Business\",\n",
    "                 4 : \"Sci/Tec\"}\n",
    "\n",
    "def predict(text, model, vocab, ngrams):\n",
    "    tokenizer = get_tokenizer(\"basic_english\")\n",
    "    with torch.no_grad():\n",
    "        text = torch.tensor([vocab[token]\n",
    "                            for token in ngrams_iterator(tokenizer(text), ngrams)])\n",
    "        output = model(text, torch.tensor([0]))\n",
    "        return output.argmax(1).item() + 1\n",
    "\n",
    "ex_text_str = \"MEMPHIS, Tenn. – Four days ago, Jon Rahm was \\\n",
    "    enduring the season’s worst weather conditions on Sunday at The \\\n",
    "    Open on his way to a closing 75 at Royal Portrush, which \\\n",
    "    considering the wind and the rain was a respectable showing. \\\n",
    "    Thursday’s first round at the WGC-FedEx St. Jude Invitational \\\n",
    "    was another story. With temperatures in the mid-80s and hardly any \\\n",
    "    wind, the Spaniard was 13 strokes better in a flawless round. \\\n",
    "    Thanks to his best putting performance on the PGA Tour, Rahm \\\n",
    "    finished with an 8-under 62 for a three-stroke lead, which \\\n",
    "    was even more impressive considering he’d never played the \\\n",
    "    front nine at TPC Southwind.\"\n",
    "\n",
    "vocab = train_dataset.get_vocab()\n",
    "model = model.to(\"cpu\")\n",
    "\n",
    "print(\"This is a %s news\" %ag_news_label[predict(ex_text_str, model, vocab, 2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}