Research_embed.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embedding Model with Instructions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_dataset_info(dataset):\n",
    "    info = dataset.info\n",
    "    dataset_name = info.dataset_name\n",
    "    splits_info = info.splits\n",
    "    features = info.features\n",
    "    print(f\"Dataset Name: {dataset_name}\")\n",
    "    print(\"Splits Info:\")\n",
    "    for split_name, split_info in splits_info.items():\n",
    "        num_examples = split_info.num_examples\n",
    "        print(f\" - Split: {split_name}, Num Examples: {num_examples}\")\n",
    "    print(\"Features:\")\n",
    "    for feature_name, feature_info in features.items():\n",
    "        print(f\" - {feature_name}: {feature_info}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import random\n",
    "from collections import defaultdict\n",
    "\n",
    "def get_dataset(dataset_name, train_size=0, test_size=0):  \n",
    "    # Load the dataset\n",
    "    dataset = load_dataset(dataset_name)\n",
    "    \n",
    "    # Access the train, test splits\n",
    "    train_dataset = dataset['train']\n",
    "    test_dataset = dataset['test']\n",
    "    \n",
    "    # Function to balance and shuffle a split\n",
    "    def shuffle(split, size):\n",
    "        if size == 0:\n",
    "            return split\n",
    "        label_to_indices = defaultdict(list)\n",
    "        for idx, example in enumerate(split):\n",
    "            label_to_indices[example['label']].append(idx)\n",
    "        \n",
    "        balanced_indices = []\n",
    "        for indices in label_to_indices.values():\n",
    "            if len(indices) >= size // len(label_to_indices):\n",
    "                balanced_indices.extend(random.sample(indices, size // len(label_to_indices)))\n",
    "            else:\n",
    "                balanced_indices.extend(indices)\n",
    "        \n",
    "        random.shuffle(balanced_indices)\n",
    "        return split.select(balanced_indices[:size])\n",
    "\n",
    "    if(train_size != 0):\n",
    "        train_dataset = train_dataset.shuffle(seed=42).select(range(train_size))\n",
    "    if(test_size != 0):\n",
    "        test_dataset = test_dataset.shuffle(seed=42).select(range(test_size))\n",
    "    # Shuffle and balance the datasets\n",
    "    # train_dataset = shuffle(train_dataset, train_size)\n",
    "    # test_dataset = shuffle(test_dataset, test_size)\n",
    "    \n",
    "    return train_dataset, test_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT - Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "from transformers.pipelines.pt_utils import KeyDataset\n",
    "import numpy as np\n",
    "\n",
    "def encode_Pipeline(model, dataset, max_length=512, use_cls=True):\n",
    "    key = \"text\"\n",
    "    data = KeyDataset(dataset, key)\n",
    "    pipe = model(data, return_tensors=True, truncation=True, padding=True, max_length=max_length)\n",
    "    embeddings=[]\n",
    "    for tensor in tqdm(pipe, desc=\"Encoding\"): \n",
    "        # Tensor Shape [batch_size, sequence_length, hidden_size]\n",
    "        if use_cls:\n",
    "            embedding = tensor[:, 0, :]\n",
    "        else:\n",
    "            embedding = tensor.mean(dim=1)\n",
    "        embeddings.append(embedding.squeeze())\n",
    "    return np.array(embeddings), np.array(dataset[\"label\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Instructor - Sentence Transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_ST(model, dataset):\n",
    "    embeddings = []\n",
    "    texts = dataset[\"text\"]\n",
    "    instructions = dataset[\"instruction\"]\n",
    "    for text, instruction in tqdm(zip(texts, instructions),total=len(dataset), desc=\"Encoding\"):\n",
    "        embedding = model.encode([[instruction, text]])\n",
    "        embeddings.append(np.array(embedding).squeeze())\n",
    "    return np.array(embeddings), np.array(dataset[\"label\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### T5 - Transformer Sentence Piece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import T5Tokenizer, T5Model\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "\n",
    "def encode_T5(dataset, key=\"text\", truncation=True, padding=True, max_length=512, use_mean_pooling=True):\n",
    "    # Check if CUDA is available and set device accordingly\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    \n",
    "    # Initialize the tokenizer and model\n",
    "    tokenizer = T5Tokenizer.from_pretrained(\"google-t5/t5-base\")\n",
    "    model = T5Model.from_pretrained(\"google-t5/t5-base\").to(device)\n",
    "    \n",
    "    embeddings = []\n",
    "    labels = []\n",
    "    \n",
    "    for data in tqdm(dataset, desc=\"Encoding text\"):\n",
    "        text = data[key]\n",
    "        label = data.get(\"label\", None)\n",
    "        \n",
    "        # Tokenize the input text\n",
    "        inputs = tokenizer(text, truncation=truncation, padding=padding, max_length=max_length, return_tensors=\"pt\")\n",
    "        inputs = {k: v.to(device) for k, v in inputs.items()}\n",
    "        \n",
    "        # Get encoder outputs\n",
    "        with torch.no_grad():\n",
    "            encoder_outputs = model.encoder(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"])\n",
    "        \n",
    "        # Pooling to get a single vector for each input\n",
    "        if use_mean_pooling:\n",
    "            attention_mask = inputs['attention_mask']\n",
    "            last_hidden_state = encoder_outputs.last_hidden_state\n",
    "            mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n",
    "            sum_embeddings = torch.sum(last_hidden_state * mask_expanded, 1)\n",
    "            sum_mask = torch.clamp(mask_expanded.sum(1), min=1e-9)\n",
    "            embedding = sum_embeddings / sum_mask\n",
    "        else:\n",
    "            embedding = encoder_outputs.last_hidden_state[:, 0, :]\n",
    "        \n",
    "        embeddings.append(embedding.cpu().numpy().flatten())  # Flatten the embeddings\n",
    "        if label is not None:\n",
    "            labels.append(label)\n",
    "    \n",
    "    return np.array(embeddings), np.array(labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Augment Instruction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mapper_affixes(example, prefix, suffix):\n",
    "    example['text'] = prefix + example['text'] + suffix\n",
    "    return example\n",
    "\n",
    "def mapper_instruct(example, instruction):\n",
    "    example['instruction'] = instruction\n",
    "    return example\n",
    "\n",
    "def augment_dataset_Affix(dataset, prefix, suffix):\n",
    "    augmented_dataset = dataset.map(lambda x: mapper_affixes(x, prefix, suffix))\n",
    "    return augmented_dataset\n",
    "\n",
    "def augment_dataset_Inst(dataset, instruction):\n",
    "    augmented_dataset = dataset.map(lambda x: mapper_instruct(x, instruction))\n",
    "    return augmented_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "\n",
    "def evaluate(method, train_embeddings, test_embeddings, train_labels, test_labels):\n",
    "    if method == \"SVM\":\n",
    "        model = SVC(kernel='linear')\n",
    "        \n",
    "    elif method == \"MLP\":\n",
    "        model = MLPClassifier(hidden_layer_sizes=(100,), max_iter=300, alpha=1e-4,\n",
    "                          solver='sgd', verbose=1, random_state=1,\n",
    "                          learning_rate_init=.1)\n",
    "\n",
    "    model.fit(train_embeddings, train_labels)\n",
    "    predicted_labels = model.predict(test_embeddings)\n",
    "    accuracy = accuracy_score(test_labels, predicted_labels)\n",
    "    print(\"Report on \" + method + \": \")\n",
    "    print(classification_report(y_true = test_labels, y_pred = predicted_labels, digits=4))\n",
    "    return round(accuracy,4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## EmbedFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "def EmbedFlow_Bert_1(train_dataset, test_dataset, evaluator, prefix, suffix=''):\n",
    "    # Load Model\n",
    "    model = pipeline(\"feature-extraction\", model=\"google-bert/bert-base-uncased\", device=0)\n",
    "    \n",
    "    train_dataset = augment_dataset_Affix(train_dataset, prefix, suffix)\n",
    "    test_dataset = augment_dataset_Affix(test_dataset, prefix, suffix)\n",
    "\n",
    "    # Embed Dataset\n",
    "    train_embeddings, train_labels = encode_Pipeline(model, train_dataset, max_length=512, use_cls=True)\n",
    "    test_embeddings, test_labels = encode_Pipeline(model, test_dataset, use_cls=True)\n",
    "\n",
    "    # Evaluate\n",
    "    accuracy = evaluate(evaluator, train_embeddings, test_embeddings, train_labels, test_labels)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def EmbedFlow_Bert_2(train_dataset, test_dataset, evaluator, prefix, suffix=''):\n",
    "    # Load Model\n",
    "    model = pipeline(\"feature-extraction\", model=\"google-bert/bert-large-uncased\", device=0)\n",
    "    \n",
    "    train_dataset = augment_dataset_Affix(train_dataset, prefix, suffix)\n",
    "    test_dataset = augment_dataset_Affix(test_dataset, prefix, suffix)\n",
    "\n",
    "    # Embed Dataset\n",
    "    train_embeddings, train_labels = encode_Pipeline(model, train_dataset, max_length=512, use_cls=False)\n",
    "    test_embeddings, test_labels = encode_Pipeline(model, test_dataset, max_length=512, use_cls=False)\n",
    "\n",
    "    # Evaluate\n",
    "    accuracy = evaluate(evaluator, train_embeddings, test_embeddings, train_labels, test_labels)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "def EmbedFlow_Instructor(train_dataset, test_dataset, evaluator, instruction):\n",
    "    # Load Model\n",
    "    model = SentenceTransformer(\"hkunlp/instructor-large\")\n",
    "    \n",
    "    # Add Instruction\n",
    "    train_dataset = augment_dataset_Inst(train_dataset, instruction)\n",
    "    test_dataset = augment_dataset_Inst(test_dataset, instruction)\n",
    "\n",
    "    # Embed Dataset\n",
    "    train_embeddings, train_labels = encode_ST(model, train_dataset)\n",
    "    test_embeddings, test_labels = encode_ST(model, test_dataset)\n",
    "    \n",
    "    # Evaluate\n",
    "    accuracy = evaluate(evaluator, train_embeddings, test_embeddings, train_labels, test_labels)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def EmbedFlow_T5(train_dataset, test_dataset, evaluator, prefix, suffix=''):\n",
    "    # Load Model\n",
    "    model = pipeline(\"feature-extraction\", model=\"t5-base\", device=0)\n",
    "\n",
    "    # Add Instruction\n",
    "    train_dataset = augment_dataset_Affix(train_dataset, prefix, suffix)\n",
    "    test_dataset = augment_dataset_Affix(test_dataset, prefix, suffix)\n",
    "\n",
    "    # Embed Dataset\n",
    "    train_embeddings, train_labels = encode_T5(train_dataset)\n",
    "    test_embeddings, test_labels = encode_T5(test_dataset)\n",
    "    \n",
    "    # Evaluate\n",
    "    accuracy = evaluate(evaluator, train_embeddings, test_embeddings, train_labels, test_labels)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def EmbedFlow_GPT(train_dataset, test_dataset, evaluator, prefix, suffix=''):\n",
    "    # Load Model\n",
    "    model = pipeline(\"feature-extraction\", model=\"openai-community/gpt2\", device=0)\n",
    "  \n",
    "    # Add Instruction\n",
    "    train_dataset = augment_dataset_Affix(train_dataset, prefix, suffix)\n",
    "    test_dataset = augment_dataset_Affix(test_dataset, prefix, suffix)\n",
    "\n",
    "    # Embed Dataset\n",
    "    train_embeddings, train_labels = encode_Pipeline(model, train_dataset, use_cls=False)\n",
    "    test_embeddings, test_labels = encode_Pipeline(model, test_dataset, use_cls=False)\n",
    "    \n",
    "    # Evaluate\n",
    "    accuracy = evaluate(evaluator, train_embeddings, test_embeddings, train_labels, test_labels)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing\n",
    "Logging: [Google Sheet](https://docs.google.com/spreadsheets/d/1iBDq7C59G6olf_of_sTF5oCY3Itj6_kImzeUl3XMpd8/edit#gid=1587051763)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "instructions_old = ['',\n",
    "          'Movie Review: ', \n",
    "          'Restaurant Review: ', \n",
    "          'Sentiment Analysis: ', \n",
    "          'User Feedback: ', \n",
    "          'Customer Experience: ',\n",
    "          'Product Review: ',\n",
    "          'Service Feedback: ',\n",
    "          'Experience at: ',\n",
    "          'Abstract: ',\n",
    "          'Research Paper Abstract: ',\n",
    "          'Paper Summary: ']\n",
    "\n",
    "instructions_general = [\n",
    "    \"Sentiment Analysis: \",\n",
    "    \"Summarize: \",\n",
    "    \"Review: \",\n",
    "    \"Evaluate: \",\n",
    "    \"Analyze: \",\n",
    "    \"Represent the text for determining the sentiment: \",\n",
    "    \"Summarize the main points of the text for a concise overview: \",\n",
    "    \"Evaluate the text to classify its sentiment and key themes: \",\n",
    "    \"Analyze the text for sentiment classification and thematic understanding: \",\n",
    "    \"Classify the sentiment and summarize the content of the given text: \"\n",
    "]\n",
    "\n",
    "instructions_imdb=[\n",
    "    \"Movie Review: \",\n",
    "    \"Film Critique: \",\n",
    "    \"Cinema Opinion: \",\n",
    "    \"Film Feedback: \",\n",
    "    \"Review Analysis: \",\n",
    "    \"Represent the Review sentence for classifying emotion as positive or negative: \",\n",
    "    \"Evaluate the sentiment of the movie review sentence: \",\n",
    "    \"Analyze the review to determine if the sentiment is positive or negative: \",\n",
    "    \"Determine the emotional tone of the review for sentiment classification: \",\n",
    "    \"Classify the given movie review as either positive or negative: \",\n",
    "]\n",
    "instructions_yelp=[\n",
    "    \"Customer Experience at: \",\n",
    "    \"Opinion on: \",\n",
    "    \"Feed back on: \",\n",
    "    \"Restaurant Review: \",\n",
    "    \"User Feedback: \",\n",
    "    \"Describe the review in terms of customer satisfaction for the given place: \",\n",
    "    \"Evaluate the review to determine the customer's overall experience: \",\n",
    "    \"Analyze the review to classify the service quality as positive or negative: \",\n",
    "    \"Represent the customer's feedback on the establishment for sentiment analysis: \",\n",
    "    \"Determine the sentiment of the review, indicating whether the experience was positive or negative: \",\n",
    "]\n",
    "\n",
    "instructions_arXiv = [\n",
    "    \"Paper Summary: \",\n",
    "    \"Overview: \",\n",
    "    \"Abstract: \",\n",
    "    \"Key points: \",\n",
    "    \"Main findings: \",\n",
    "    \"Please read the following research paper abstract and summarize the key points and findings discussed: \",\n",
    "    \"Analyze the following abstract from a research paper and provide a detailed overview of its main ideas and conclusions: \",\n",
    "    \"Review the following abstract of a research paper and highlight the primary methodologies and results presented: \",\n",
    "    \"Examine the following abstract from a scientific paper and summarize the main topics and discoveries described: \",\n",
    "    \"Process the following research paper abstract and provide a concise summary of the core themes and contributions of the study: \"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dataset: stanfordnlp/imdb, instruction: 'Sentiment Analysis: '\n",
      "Bert\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Encoding: 100%|██████████| 1000/1000 [00:06<00:00, 163.59it/s]\n",
      "Encoding: 100%|██████████| 1000/1000 [00:07<00:00, 129.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Report on SVM: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7960    0.7773    0.7866       512\n",
      "           1     0.7720    0.7910    0.7814       488\n",
      "\n",
      "    accuracy                         0.7840      1000\n",
      "   macro avg     0.7840    0.7842    0.7840      1000\n",
      "weighted avg     0.7843    0.7840    0.7840      1000\n",
      "\n",
      "Bert2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Encoding:  72%|███████▏  | 717/1000 [00:14<00:05, 51.18it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[85], line 37\u001b[0m\n\u001b[0;32m     34\u001b[0m data[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBert\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mappend(accB1)\n\u001b[0;32m     36\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBert2\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m---> 37\u001b[0m accB2 \u001b[38;5;241m=\u001b[39m \u001b[43mEmbedFlow_Bert_2\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevaluator\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minstruction\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     38\u001b[0m data[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBert-large\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mappend(accB2)\n\u001b[0;32m     40\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInstructor\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "Cell \u001b[1;32mIn[80], line 9\u001b[0m, in \u001b[0;36mEmbedFlow_Bert_2\u001b[1;34m(train_dataset, test_dataset, evaluator, prefix, suffix)\u001b[0m\n\u001b[0;32m      6\u001b[0m test_dataset \u001b[38;5;241m=\u001b[39m augment_dataset_Affix(test_dataset, prefix, suffix)\n\u001b[0;32m      8\u001b[0m \u001b[38;5;66;03m# Embed Dataset\u001b[39;00m\n\u001b[1;32m----> 9\u001b[0m train_embeddings, train_labels \u001b[38;5;241m=\u001b[39m \u001b[43mencode_Pipeline\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_length\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m512\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muse_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m     10\u001b[0m test_embeddings, test_labels \u001b[38;5;241m=\u001b[39m encode_Pipeline(model, test_dataset, max_length\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m512\u001b[39m, use_cls\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m     12\u001b[0m \u001b[38;5;66;03m# Evaluate\u001b[39;00m\n",
      "Cell \u001b[1;32mIn[73], line 10\u001b[0m, in \u001b[0;36mencode_Pipeline\u001b[1;34m(model, dataset, max_length, use_cls)\u001b[0m\n\u001b[0;32m      8\u001b[0m pipe \u001b[38;5;241m=\u001b[39m model(data, return_tensors\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, truncation\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, padding\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, max_length\u001b[38;5;241m=\u001b[39mmax_length)\n\u001b[0;32m      9\u001b[0m embeddings\u001b[38;5;241m=\u001b[39m[]\n\u001b[1;32m---> 10\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m tensor \u001b[38;5;129;01min\u001b[39;00m tqdm(pipe, desc\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncoding\u001b[39m\u001b[38;5;124m\"\u001b[39m): \n\u001b[0;32m     11\u001b[0m     \u001b[38;5;66;03m# Tensor Shape [batch_size, sequence_length, hidden_size]\u001b[39;00m\n\u001b[0;32m     12\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m use_cls:\n\u001b[0;32m     13\u001b[0m         embedding \u001b[38;5;241m=\u001b[39m tensor[:, \u001b[38;5;241m0\u001b[39m, :]\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\tqdm\\std.py:1181\u001b[0m, in \u001b[0;36mtqdm.__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1178\u001b[0m time \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_time\n\u001b[0;32m   1180\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 1181\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m obj \u001b[38;5;129;01min\u001b[39;00m iterable:\n\u001b[0;32m   1182\u001b[0m         \u001b[38;5;28;01myield\u001b[39;00m obj\n\u001b[0;32m   1183\u001b[0m         \u001b[38;5;66;03m# Update and possibly print the progressbar.\u001b[39;00m\n\u001b[0;32m   1184\u001b[0m         \u001b[38;5;66;03m# Note: does not call self.update(1) for speed optimisation.\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\pipelines\\pt_utils.py:124\u001b[0m, in \u001b[0;36mPipelineIterator.__next__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    121\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mloader_batch_item()\n\u001b[0;32m    123\u001b[0m \u001b[38;5;66;03m# We're out of items within a batch\u001b[39;00m\n\u001b[1;32m--> 124\u001b[0m item \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miterator)\n\u001b[0;32m    125\u001b[0m processed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minfer(item, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mparams)\n\u001b[0;32m    126\u001b[0m \u001b[38;5;66;03m# We now have a batch of \"inferred things\".\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\pipelines\\pt_utils.py:125\u001b[0m, in \u001b[0;36mPipelineIterator.__next__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    123\u001b[0m \u001b[38;5;66;03m# We're out of items within a batch\u001b[39;00m\n\u001b[0;32m    124\u001b[0m item \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miterator)\n\u001b[1;32m--> 125\u001b[0m processed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minfer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mitem\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    126\u001b[0m \u001b[38;5;66;03m# We now have a batch of \"inferred things\".\u001b[39;00m\n\u001b[0;32m    127\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mloader_batch_size \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    128\u001b[0m     \u001b[38;5;66;03m# Try to infer the size of the batch\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\pipelines\\base.py:1150\u001b[0m, in \u001b[0;36mPipeline.forward\u001b[1;34m(self, model_inputs, **forward_params)\u001b[0m\n\u001b[0;32m   1148\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m inference_context():\n\u001b[0;32m   1149\u001b[0m         model_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_ensure_tensor_on_device(model_inputs, device\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[1;32m-> 1150\u001b[0m         model_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mforward_params\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1151\u001b[0m         model_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_ensure_tensor_on_device(model_outputs, device\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mdevice(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n\u001b[0;32m   1152\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\pipelines\\feature_extraction.py:64\u001b[0m, in \u001b[0;36mFeatureExtractionPipeline._forward\u001b[1;34m(self, model_inputs)\u001b[0m\n\u001b[0;32m     63\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_forward\u001b[39m(\u001b[38;5;28mself\u001b[39m, model_inputs):\n\u001b[1;32m---> 64\u001b[0m     model_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m model_outputs\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:1137\u001b[0m, in \u001b[0;36mBertModel.forward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m   1130\u001b[0m \u001b[38;5;66;03m# Prepare head mask if needed\u001b[39;00m\n\u001b[0;32m   1131\u001b[0m \u001b[38;5;66;03m# 1.0 in head_mask indicate we keep the head\u001b[39;00m\n\u001b[0;32m   1132\u001b[0m \u001b[38;5;66;03m# attention_probs has shape bsz x n_heads x N x N\u001b[39;00m\n\u001b[0;32m   1133\u001b[0m \u001b[38;5;66;03m# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]\u001b[39;00m\n\u001b[0;32m   1134\u001b[0m \u001b[38;5;66;03m# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]\u001b[39;00m\n\u001b[0;32m   1135\u001b[0m head_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_head_mask(head_mask, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_layers)\n\u001b[1;32m-> 1137\u001b[0m encoder_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1138\u001b[0m \u001b[43m    \u001b[49m\u001b[43membedding_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1139\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1140\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1141\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1142\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_extended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1143\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1144\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1145\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1146\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1147\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1148\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1149\u001b[0m sequence_output \u001b[38;5;241m=\u001b[39m encoder_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m   1150\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler(sequence_output) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:690\u001b[0m, in \u001b[0;36mBertEncoder.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m    679\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[0;32m    680\u001b[0m         layer_module\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[0;32m    681\u001b[0m         hidden_states,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    687\u001b[0m         output_attentions,\n\u001b[0;32m    688\u001b[0m     )\n\u001b[0;32m    689\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 690\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mlayer_module\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    691\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    692\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    693\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlayer_head_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    694\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    695\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    696\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    697\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    698\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    700\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m    701\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:580\u001b[0m, in \u001b[0;36mBertLayer.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m    568\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[0;32m    569\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    570\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    577\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[0;32m    578\u001b[0m     \u001b[38;5;66;03m# decoder uni-directional self-attention cached key/values tuple is at positions 1,2\u001b[39;00m\n\u001b[0;32m    579\u001b[0m     self_attn_past_key_value \u001b[38;5;241m=\u001b[39m past_key_value[:\u001b[38;5;241m2\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m past_key_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m--> 580\u001b[0m     self_attention_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattention\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    581\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    582\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    583\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    584\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    585\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mself_attn_past_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    586\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    587\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m self_attention_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m    589\u001b[0m     \u001b[38;5;66;03m# if decoder, the last output is tuple of self-attn cache\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:510\u001b[0m, in \u001b[0;36mBertAttention.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m    500\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[0;32m    501\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    502\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    508\u001b[0m     output_attentions: Optional[\u001b[38;5;28mbool\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[0;32m    509\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[1;32m--> 510\u001b[0m     self_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mself\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    511\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    512\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    513\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    514\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    515\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    516\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    517\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    518\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    519\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput(self_outputs[\u001b[38;5;241m0\u001b[39m], hidden_states)\n\u001b[0;32m    520\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m (attention_output,) \u001b[38;5;241m+\u001b[39m self_outputs[\u001b[38;5;241m1\u001b[39m:]  \u001b[38;5;66;03m# add attentions if we output them\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:408\u001b[0m, in \u001b[0;36mBertSdpaSelfAttention.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m    406\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    407\u001b[0m     key_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtranspose_for_scores(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkey(current_states))\n\u001b[1;32m--> 408\u001b[0m     value_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtranspose_for_scores(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalue\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcurrent_states\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m    409\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m past_key_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_cross_attention:\n\u001b[0;32m    410\u001b[0m         key_layer \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mcat([past_key_value[\u001b[38;5;241m0\u001b[39m], key_layer], dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\xsyyy\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\linear.py:116\u001b[0m, in \u001b[0;36mLinear.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 116\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "instructions_map = {\n",
    "    'stanfordnlp/imdb': instructions_imdb,\n",
    "    'yelp_review_full': instructions_yelp,\n",
    "    'Voice49/arXiv-Abstract-Label-20k': instructions_arXiv\n",
    "}\n",
    "\n",
    "datasets = ['stanfordnlp/imdb', \n",
    "            'yelp_review_full',\n",
    "            'Voice49/arXiv-Abstract-Label-20k']\n",
    "\n",
    "evaluator = ['SVM', 'MLP']\n",
    "\n",
    "train, test = get_dataset(datasets[0], train_size=1000, test_size=1000)\n",
    "\n",
    "# Data structure to record accuracies\n",
    "data = {\n",
    "    \"Instructions\": [\"None\", \"GS1\", \"GS2\", \"GS3\", \"GS4\", \"GS5\", \"GL1\", \"GL2\", \"GL3\", \"GL4\", \"GL5\", \n",
    "                     \"S1\", \"S2\", \"S3\", \"S4\", \"S5\", \"L1\", \"L2\", \"L3\", \"L4\", \"L5\"],\n",
    "    \"Bert\": [],\n",
    "    \"Bert-large\": [],\n",
    "    \"Instructor\": [],\n",
    "    \"T5\": [],\n",
    "    \"GPT2\": []\n",
    "}\n",
    "\n",
    "# Loop last dataset\n",
    "for dataset in datasets[:1]:\n",
    "    instructions = instructions_general + instructions_map[dataset]\n",
    "    for instruction in instructions:\n",
    "        print(f\"Processing dataset: {dataset}, instruction: '{instruction}'\")\n",
    "\n",
    "        print(\"Bert\")\n",
    "        accB1 = EmbedFlow_Bert_1(train, test, evaluator[0], instruction)\n",
    "        data[\"Bert\"].append(accB1)\n",
    "\n",
    "        print(\"Bert2\")\n",
    "        accB2 = EmbedFlow_Bert_2(train, test, evaluator[0], instruction)\n",
    "        data[\"Bert-large\"].append(accB2)\n",
    "\n",
    "        print(\"Instructor\")\n",
    "        accI = EmbedFlow_Instructor(train, test, evaluator[0], instruction)\n",
    "        data[\"Instructor\"].append(accI)\n",
    "\n",
    "        print(\"T5\")\n",
    "        accT = EmbedFlow_T5(train, test, evaluator[0], instruction)\n",
    "        data[\"T5\"].append(accT)\n",
    "\n",
    "        print(\"GPT2\")\n",
    "        accG = EmbedFlow_GPT(train, test, evaluator[0], instruction)\n",
    "        data[\"GPT2\"].append(accG)\n",
    "\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# Display the DataFrame\n",
    "# print(df)\n",
    "\n",
    "# Visualizing the data using appropriate plots\n",
    "plt.figure(figsize=(14, 8))\n",
    "\n",
    "for column in df.columns[1:]:\n",
    "    plt.plot(df[\"Instructions\"], df[column], marker='o', label=column)\n",
    "\n",
    "plt.title(\"Classifier Accuracy on Text Embedding Models with Instructions\")\n",
    "plt.xlabel(\"Instructions\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.legend(loc='upper center', bbox_to_anchor=(1.02, 1))\n",
    "plt.grid(True)\n",
    "plt.xticks(rotation=45)\n",
    "plt.tight_layout()\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate statistics for general, specific, and combined instructions\n",
    "statistics = {}\n",
    "for column in df.columns[1:]:\n",
    "    general_instructions = df[column][1:11]  # GS1 to GL5\n",
    "    specific_instructions = df[column][11:]  # S1 to L5\n",
    "    combined_instructions = df[column][1:]   # All instructions (GS1 to L5)\n",
    "    without_instructions = df[column][0]\n",
    "    \n",
    "    mean_general = general_instructions.mean()\n",
    "    std_general = general_instructions.std()\n",
    "    mean_specific = specific_instructions.mean()\n",
    "    std_specific = specific_instructions.std()\n",
    "    mean_combined = combined_instructions.mean()\n",
    "    std_combined = combined_instructions.std()\n",
    "    \n",
    "    change_general = mean_general - without_instructions\n",
    "    change_specific = mean_specific - without_instructions\n",
    "    change_combined = mean_combined - without_instructions\n",
    "    \n",
    "    statistics[column] = {\n",
    "        'Base': without_instructions,\n",
    "        'Mean (G)': mean_general,\n",
    "        'Mean (S)': mean_specific,\n",
    "        'Mean (All)': mean_combined,\n",
    "        'Std (G)': std_general,\n",
    "        'Std (S)': std_specific,\n",
    "        'Std (All)': std_combined,\n",
    "        'Change (G)': change_general,\n",
    "        'Change (S)': change_specific,\n",
    "        'Change (All)': change_combined\n",
    "    }\n",
    "\n",
    "# Display the statistics\n",
    "stats_df = pd.DataFrame(statistics).T\n",
    "print(stats_df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "embed",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}