DNN_Pesos&Amostragem.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dGfJfeJv35Kv",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2893,
     "status": "ok",
     "timestamp": 1675097614830,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "dGfJfeJv35Kv",
    "outputId": "693c9ce3-7174-4a33-b936-3936b3c49e4e"
   },
   "outputs": [],
   "source": [
    "data_path = \"../data/70-15-15\"\n",
    "model_path = \"../models\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0oK9W-U39Yd7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4568,
     "status": "ok",
     "timestamp": 1675097619392,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "0oK9W-U39Yd7",
    "outputId": "4d1ead00-ff1c-44d1-e2b3-fdfcf6b96cf2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow in /opt/conda/lib/python3.9/site-packages (2.6.2)\n",
      "Requirement already satisfied: clang~=5.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (5.0)\n",
      "Requirement already satisfied: h5py~=3.1.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (3.1.0)\n",
      "Requirement already satisfied: termcolor~=1.1.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.1.0)\n",
      "Requirement already satisfied: keras-preprocessing~=1.1.2 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.1.2)\n",
      "Requirement already satisfied: gast==0.4.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (0.4.0)\n",
      "Requirement already satisfied: keras<2.7,>=2.6.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (2.6.0)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (3.19.3)\n",
      "Requirement already satisfied: tensorflow-estimator<2.7,>=2.6.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (2.6.0)\n",
      "Requirement already satisfied: astunparse~=1.6.3 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers~=1.12.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.12)\n",
      "Requirement already satisfied: absl-py~=0.10 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (0.15.0)\n",
      "Requirement already satisfied: numpy~=1.19.2 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.19.5)\n",
      "Requirement already satisfied: opt-einsum~=3.3.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: google-pasta~=0.2 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: wrapt~=1.12.1 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.12.1)\n",
      "Requirement already satisfied: typing-extensions~=3.7.4 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (3.7.4.3)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.37.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.43.0)\n",
      "Requirement already satisfied: tensorboard<2.7,>=2.6.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (2.6.0)\n",
      "Requirement already satisfied: six~=1.15.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.15.0)\n",
      "Requirement already satisfied: wheel~=0.35 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (0.37.1)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow) (2.27.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow) (1.8.1)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow) (1.35.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow) (2.0.2)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow) (3.3.6)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow) (59.8.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow) (0.4.6)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow) (0.6.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.9/site-packages (from google-auth<2,>=1.6.3->tensorboard<2.7,>=2.6.0->tensorflow) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.9/site-packages (from google-auth<2,>=1.6.3->tensorboard<2.7,>=2.6.0->tensorflow) (4.8)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.9/site-packages (from google-auth<2,>=1.6.3->tensorboard<2.7,>=2.6.0->tensorflow) (4.2.4)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.9/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.7,>=2.6.0->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.9/site-packages (from markdown>=2.6.8->tensorboard<2.7,>=2.6.0->tensorflow) (4.10.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.7,>=2.6.0->tensorflow) (2021.10.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.7,>=2.6.0->tensorflow) (3.3)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.7,>=2.6.0->tensorflow) (2.0.10)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.7,>=2.6.0->tensorflow) (1.26.8)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.9/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.7,>=2.6.0->tensorflow) (3.7.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.7,>=2.6.0->tensorflow) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.9/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.7,>=2.6.0->tensorflow) (3.2.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow\n",
    "#!pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dbd010dd-8f29-48b1-9e89-2cab82e6ea60",
   "metadata": {
    "executionInfo": {
     "elapsed": 1080,
     "status": "ok",
     "timestamp": 1675097620466,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "dbd010dd-8f29-48b1-9e89-2cab82e6ea60"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "85187dff-a49f-4dab-acc5-0479a7694cb8",
   "metadata": {
    "executionInfo": {
     "elapsed": 682,
     "status": "ok",
     "timestamp": 1675097621145,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "85187dff-a49f-4dab-acc5-0479a7694cb8"
   },
   "outputs": [],
   "source": [
    "treino=pd.read_csv(data_path+'/comunic_treino_aug.csv', encoding='utf-8')\n",
    "teste=pd.read_csv(data_path+'/comunic_teste.csv', encoding='utf-8')\n",
    "validacao=pd.read_csv(data_path+'/comunic_valid.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "gFFnKgbUdaY0",
   "metadata": {
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1675097621146,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "gFFnKgbUdaY0"
   },
   "outputs": [],
   "source": [
    "def TFIDF(X_train, X_test,MAX_NB_WORDS=4000):\n",
    "    vectorizer_x = TfidfVectorizer(max_features=MAX_NB_WORDS)\n",
    "    X_train = vectorizer_x.fit_transform(X_train).toarray()\n",
    "    X_test = vectorizer_x.transform(X_test).toarray()\n",
    "    print(\"tf-idf with\",str(np.array(X_train).shape[1]),\"features\")\n",
    "    return (X_train,X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "734d7b64-95e4-4fc0-840f-f4f66e0ce340",
   "metadata": {
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1675097621146,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "734d7b64-95e4-4fc0-840f-f4f66e0ce340"
   },
   "outputs": [],
   "source": [
    "def make_tfidf(coluna):\n",
    "    vectorizer = TfidfVectorizer()\n",
    "    vectors = np.asarray(vectorizer.fit_transform(coluna).todense())\n",
    "    return vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "01f1e27a-ace3-48f6-a348-9466c2acfbd4",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1675097621147,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "01f1e27a-ace3-48f6-a348-9466c2acfbd4",
    "outputId": "ca87d1ac-ab75-43b8-b478-0c9a7f521efd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf-idf with 4000 features\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = TFIDF(treino[\"text\"], teste[\"text\"])\n",
    "Y_train=treino.label\n",
    "Y_test=teste.label\n",
    "X_val=make_tfidf(validacao.text)\n",
    "Y_val=validacao.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8ac063e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 1.1276923076923078,\n",
       " 1: 1.1276923076923078,\n",
       " 2: 1.1276923076923078,\n",
       " 3: 1.1276923076923078,\n",
       " 4: 0.42394447657605555,\n",
       " 5: 1.1276923076923078,\n",
       " 6: 1.1276923076923078,\n",
       " 7: 1.1276923076923078,\n",
       " 8: 1.1276923076923078,\n",
       " 9: 1.1276923076923078,\n",
       " 10: 1.1276923076923078,\n",
       " 11: 1.1276923076923078,\n",
       " 12: 1.1276923076923078}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "class_weights = compute_class_weight(class_weight = \"balanced\", classes= np.unique(treino.label), y= treino.label)\n",
    "class_weights = dict(zip(np.unique(treino.label), class_weights))\n",
    "class_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6932765d-6b4a-45bf-97c3-c7751a017724",
   "metadata": {
    "id": "6932765d-6b4a-45bf-97c3-c7751a017724"
   },
   "source": [
    "# Definição do modelo de classificacao DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "PuKm4srVFMLL",
   "metadata": {
    "executionInfo": {
     "elapsed": 2489,
     "status": "ok",
     "timestamp": 1675097623630,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "PuKm4srVFMLL"
   },
   "outputs": [],
   "source": [
    "from tensorflow.python.keras import models\n",
    "from tensorflow.python.keras import layers\n",
    "from tensorflow.python.keras import losses\n",
    "from tensorflow.python.keras import optimizers\n",
    "\n",
    "from tensorflow.python.keras.callbacks import EarlyStopping\n",
    "\n",
    "early_stopping =  EarlyStopping(\n",
    "    monitor='val_accuracy', \n",
    "    patience=4\n",
    ")\n",
    "\n",
    "def Build_Model_DNN_Text(shape, nClasses, dropout=0.4):\n",
    "    model = models.Sequential()\n",
    "    model.add(layers.Dense(4000,input_dim=shape,activation='relu'))\n",
    "    model.add(layers.Dense(shape,activation='relu'))\n",
    "    model.add(layers.Dropout(dropout))\n",
    "    model.add(layers.Dense(1024,activation='relu'))\n",
    "    model.add(layers.Dropout(dropout))\n",
    "    model.add(layers.Dense(128,activation='relu'))\n",
    "    model.add(layers.Dropout(dropout))    \n",
    "    model.add(layers.Dense(1024,activation='relu'))\n",
    "    model.add(layers.Dropout(dropout))    \n",
    "    model.add(layers.Dense(nClasses, activation='sigmoid'))\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                  optimizer='Adam',\n",
    "                  metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "Gc8dcyskKuJ3",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 132261,
     "status": "ok",
     "timestamp": 1675097767282,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "Gc8dcyskKuJ3",
    "outputId": "d794d2cd-52ac-4e6d-9c34-6e523e1163f9"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-07 00:17:27.947129: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-07 00:17:27.951120: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-07 00:17:27.951246: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-07 00:17:27.951823: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-03-07 00:17:27.952266: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-07 00:17:27.952403: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-07 00:17:27.952510: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-07 00:17:28.285045: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-07 00:17:28.285216: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-07 00:17:28.285329: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-07 00:17:28.285425: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 5150 MB memory:  -> device: 0, name: Quadro RTX 4000, pci bus id: 0000:01:00.0, compute capability: 7.5\n",
      "2023-03-07 00:17:28.451662: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 4000)              16004000  \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 4000)              16004000  \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 4000)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1024)              4097024   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 128)               131200    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1024)              132096    \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 13)                13325     \n",
      "=================================================================\n",
      "Total params: 36,381,645\n",
      "Trainable params: 36,381,645\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/50\n",
      "23/23 - 1s - loss: 2.4947 - accuracy: 0.1088 - val_loss: 2.4126 - val_accuracy: 0.1564\n",
      "Epoch 2/50\n",
      "23/23 - 0s - loss: 1.0878 - accuracy: 0.6132 - val_loss: 1.0291 - val_accuracy: 0.7325\n",
      "Epoch 3/50\n",
      "23/23 - 0s - loss: 0.2324 - accuracy: 0.9161 - val_loss: 0.9784 - val_accuracy: 0.7942\n",
      "Epoch 4/50\n",
      "23/23 - 0s - loss: 0.0705 - accuracy: 0.9789 - val_loss: 1.2851 - val_accuracy: 0.7778\n",
      "Epoch 5/50\n",
      "23/23 - 0s - loss: 0.0297 - accuracy: 0.9894 - val_loss: 1.3079 - val_accuracy: 0.8025\n",
      "Epoch 6/50\n",
      "23/23 - 0s - loss: 0.0246 - accuracy: 0.9949 - val_loss: 1.3438 - val_accuracy: 0.7860\n",
      "Epoch 7/50\n",
      "23/23 - 0s - loss: 0.0085 - accuracy: 0.9983 - val_loss: 1.7160 - val_accuracy: 0.7860\n",
      "Epoch 8/50\n",
      "23/23 - 0s - loss: 0.0109 - accuracy: 0.9980 - val_loss: 1.3778 - val_accuracy: 0.8148\n",
      "Epoch 9/50\n",
      "23/23 - 0s - loss: 0.0117 - accuracy: 0.9969 - val_loss: 1.5798 - val_accuracy: 0.7984\n",
      "Epoch 10/50\n",
      "23/23 - 0s - loss: 0.0137 - accuracy: 0.9973 - val_loss: 1.8276 - val_accuracy: 0.7942\n",
      "Epoch 11/50\n",
      "23/23 - 0s - loss: 0.0054 - accuracy: 0.9986 - val_loss: 1.6588 - val_accuracy: 0.8066\n",
      "Epoch 12/50\n",
      "23/23 - 0s - loss: 0.0084 - accuracy: 0.9983 - val_loss: 1.8591 - val_accuracy: 0.7984\n",
      "Epoch 13/50\n",
      "23/23 - 0s - loss: 0.0068 - accuracy: 0.9973 - val_loss: 1.7346 - val_accuracy: 0.8066\n",
      "Epoch 14/50\n",
      "23/23 - 0s - loss: 0.0159 - accuracy: 0.9983 - val_loss: 1.8848 - val_accuracy: 0.8025\n",
      "Epoch 15/50\n",
      "23/23 - 0s - loss: 0.0088 - accuracy: 0.9980 - val_loss: 1.6584 - val_accuracy: 0.8107\n",
      "Epoch 16/50\n",
      "23/23 - 0s - loss: 0.0096 - accuracy: 0.9966 - val_loss: 1.8984 - val_accuracy: 0.7942\n",
      "Epoch 17/50\n",
      "23/23 - 0s - loss: 0.0091 - accuracy: 0.9966 - val_loss: 1.8320 - val_accuracy: 0.7819\n",
      "Epoch 18/50\n",
      "23/23 - 0s - loss: 0.0117 - accuracy: 0.9969 - val_loss: 2.5137 - val_accuracy: 0.7572\n",
      "Epoch 19/50\n",
      "23/23 - 0s - loss: 0.0178 - accuracy: 0.9959 - val_loss: 2.3177 - val_accuracy: 0.7819\n",
      "Epoch 20/50\n",
      "23/23 - 0s - loss: 0.0237 - accuracy: 0.9949 - val_loss: 1.9818 - val_accuracy: 0.7737\n",
      "Epoch 21/50\n",
      "23/23 - 0s - loss: 0.0195 - accuracy: 0.9956 - val_loss: 2.2250 - val_accuracy: 0.7942\n",
      "Epoch 22/50\n",
      "23/23 - 0s - loss: 0.0222 - accuracy: 0.9959 - val_loss: 1.9056 - val_accuracy: 0.7819\n",
      "Epoch 23/50\n",
      "23/23 - 0s - loss: 0.0056 - accuracy: 0.9983 - val_loss: 1.8529 - val_accuracy: 0.7654\n",
      "Epoch 24/50\n",
      "23/23 - 0s - loss: 0.0075 - accuracy: 0.9980 - val_loss: 1.9487 - val_accuracy: 0.7860\n",
      "Epoch 25/50\n",
      "23/23 - 0s - loss: 0.0051 - accuracy: 0.9993 - val_loss: 1.9694 - val_accuracy: 0.8025\n",
      "Epoch 26/50\n",
      "23/23 - 0s - loss: 0.0183 - accuracy: 0.9973 - val_loss: 2.0057 - val_accuracy: 0.7695\n",
      "Epoch 27/50\n",
      "23/23 - 0s - loss: 0.0105 - accuracy: 0.9983 - val_loss: 2.7304 - val_accuracy: 0.7654\n",
      "Epoch 28/50\n",
      "23/23 - 0s - loss: 0.0133 - accuracy: 0.9976 - val_loss: 2.1912 - val_accuracy: 0.7572\n",
      "Epoch 29/50\n",
      "23/23 - 0s - loss: 0.0213 - accuracy: 0.9962 - val_loss: 2.6116 - val_accuracy: 0.7695\n",
      "Epoch 30/50\n",
      "23/23 - 0s - loss: 0.0182 - accuracy: 0.9976 - val_loss: 2.0895 - val_accuracy: 0.7819\n",
      "Epoch 31/50\n",
      "23/23 - 0s - loss: 0.0119 - accuracy: 0.9980 - val_loss: 2.2652 - val_accuracy: 0.7572\n",
      "Epoch 32/50\n",
      "23/23 - 0s - loss: 0.0179 - accuracy: 0.9945 - val_loss: 2.4035 - val_accuracy: 0.7737\n",
      "Epoch 33/50\n",
      "23/23 - 0s - loss: 0.0329 - accuracy: 0.9962 - val_loss: 2.8015 - val_accuracy: 0.7613\n",
      "Epoch 34/50\n",
      "23/23 - 0s - loss: 0.0201 - accuracy: 0.9973 - val_loss: 2.8892 - val_accuracy: 0.7778\n",
      "Epoch 35/50\n",
      "23/23 - 0s - loss: 0.0375 - accuracy: 0.9969 - val_loss: 3.0905 - val_accuracy: 0.7654\n",
      "Epoch 36/50\n",
      "23/23 - 0s - loss: 0.0155 - accuracy: 0.9973 - val_loss: 2.1333 - val_accuracy: 0.7860\n",
      "Epoch 37/50\n",
      "23/23 - 0s - loss: 0.0019 - accuracy: 0.9997 - val_loss: 2.0755 - val_accuracy: 0.7942\n",
      "Epoch 38/50\n",
      "23/23 - 0s - loss: 0.0063 - accuracy: 0.9993 - val_loss: 2.9844 - val_accuracy: 0.7819\n",
      "Epoch 39/50\n",
      "23/23 - 0s - loss: 0.0046 - accuracy: 0.9993 - val_loss: 2.6290 - val_accuracy: 0.7984\n",
      "Epoch 40/50\n",
      "23/23 - 0s - loss: 0.0055 - accuracy: 0.9990 - val_loss: 2.5685 - val_accuracy: 0.8025\n",
      "Epoch 41/50\n",
      "23/23 - 0s - loss: 0.0035 - accuracy: 0.9990 - val_loss: 2.4796 - val_accuracy: 0.7984\n",
      "Epoch 42/50\n",
      "23/23 - 0s - loss: 0.0074 - accuracy: 0.9983 - val_loss: 2.5003 - val_accuracy: 0.7942\n",
      "Epoch 43/50\n",
      "23/23 - 0s - loss: 0.0030 - accuracy: 0.9993 - val_loss: 2.7850 - val_accuracy: 0.7901\n",
      "Epoch 44/50\n",
      "23/23 - 0s - loss: 0.0082 - accuracy: 0.9990 - val_loss: 2.7267 - val_accuracy: 0.7901\n",
      "Epoch 45/50\n",
      "23/23 - 0s - loss: 0.0024 - accuracy: 0.9993 - val_loss: 2.5414 - val_accuracy: 0.7778\n",
      "Epoch 46/50\n",
      "23/23 - 0s - loss: 0.0317 - accuracy: 0.9969 - val_loss: 2.7421 - val_accuracy: 0.7942\n",
      "Epoch 47/50\n",
      "23/23 - 0s - loss: 0.0068 - accuracy: 0.9990 - val_loss: 2.7015 - val_accuracy: 0.7654\n",
      "Epoch 48/50\n",
      "23/23 - 0s - loss: 0.0116 - accuracy: 0.9983 - val_loss: 2.7047 - val_accuracy: 0.7901\n",
      "Epoch 49/50\n",
      "23/23 - 0s - loss: 0.0024 - accuracy: 0.9993 - val_loss: 3.0463 - val_accuracy: 0.7819\n",
      "Epoch 50/50\n",
      "23/23 - 0s - loss: 0.0072 - accuracy: 0.9990 - val_loss: 3.3471 - val_accuracy: 0.7860\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.87      0.81        15\n",
      "           1       0.58      0.83      0.68        18\n",
      "           2       1.00      0.33      0.50         3\n",
      "           3       0.75      0.43      0.55         7\n",
      "           4       0.90      0.96      0.93       114\n",
      "           5       0.50      0.78      0.61         9\n",
      "           6       0.00      0.00      0.00         4\n",
      "           7       0.80      0.29      0.42        14\n",
      "           8       1.00      0.75      0.86         4\n",
      "           9       0.74      0.64      0.68        22\n",
      "          10       0.50      0.57      0.53         7\n",
      "          11       1.00      0.25      0.40         4\n",
      "          12       0.75      0.82      0.78        22\n",
      "\n",
      "    accuracy                           0.79       243\n",
      "   macro avg       0.71      0.58      0.60       243\n",
      "weighted avg       0.79      0.79      0.77       243\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/opt/conda/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/opt/conda/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "model_DNN = Build_Model_DNN_Text(X_train.shape[1], 13)\n",
    "model_DNN.fit(X_train, Y_train,\n",
    "                              validation_data=(X_test, Y_test),\n",
    "                              epochs=50,\n",
    "                              batch_size=128,\n",
    "                              verbose=2,\n",
    "                              class_weight=class_weights)\n",
    "                              #callbacks=[early_stopping])\n",
    "predicted = model_DNN.predict(X_test)\n",
    "Y_pred = np.argmax(predicted, axis=1)\n",
    "print(metrics.classification_report(Y_test, Y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3b1b126-a5c7-4caf-8a94-77ea31b690f1",
   "metadata": {
    "id": "c3b1b126-a5c7-4caf-8a94-77ea31b690f1"
   },
   "source": [
    "### Métricas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3810f4d0-4f8f-424e-b71c-9cd3fb09e71e",
   "metadata": {
    "executionInfo": {
     "elapsed": 387,
     "status": "ok",
     "timestamp": 1675097825090,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "3810f4d0-4f8f-424e-b71c-9cd3fb09e71e"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score, roc_auc_score, accuracy_score, precision_score, recall_score, hamming_loss\n",
    "precision = precision_score(Y_test, Y_pred,average='micro')\n",
    "recall = recall_score(Y_test, Y_pred,average='micro')\n",
    "f1_micro_average = f1_score(y_true=Y_test, y_pred=Y_pred, average='micro')\n",
    "#roc_auc = roc_auc_score(Y_test, Y_pred, multi_class ='ovr', average = 'micro')\n",
    "accuracy = accuracy_score(Y_test, Y_pred)\n",
    "hloss = hamming_loss(Y_test, Y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "503cc683",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 433,
     "status": "ok",
     "timestamp": 1675097833896,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "503cc683",
    "outputId": "6720b7f7-483b-4245-fa41-102ed3627d6c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acurácia: 0.7901234567901234\n",
      "Precisão: 0.7901234567901234\n",
      "Recall: 0.7901234567901234\n",
      "F1: 0.7901234567901234\n",
      "Hamming Loss: 0.20987654320987653\n"
     ]
    }
   ],
   "source": [
    "print(\"Acurácia:\",accuracy)\n",
    "print(\"Precisão:\",precision)\n",
    "print(\"Recall:\",recall)\n",
    "print(\"F1:\",f1_micro_average)\n",
    "#print(\"Roc_auc:\",roc_auc)\n",
    "print(\"Hamming Loss:\",hloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4871accb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}