DNN_Amostragem.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dGfJfeJv35Kv",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2893,
     "status": "ok",
     "timestamp": 1675097614830,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "dGfJfeJv35Kv",
    "outputId": "693c9ce3-7174-4a33-b936-3936b3c49e4e"
   },
   "outputs": [],
   "source": [
    "data_path = \"../data/70-15-15\"\n",
    "model_path = \"../models\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0oK9W-U39Yd7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4568,
     "status": "ok",
     "timestamp": 1675097619392,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "0oK9W-U39Yd7",
    "outputId": "4d1ead00-ff1c-44d1-e2b3-fdfcf6b96cf2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow in /opt/conda/lib/python3.9/site-packages (2.11.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.43.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.12.1)\n",
      "Requirement already satisfied: flatbuffers>=2.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (23.3.3)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.9/site-packages (from tensorflow) (21.3)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.1.0)\n",
      "Requirement already satisfied: tensorboard<2.12,>=2.11 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (2.11.2)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (3.1.0)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (0.4.0)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (15.0.6.1)\n",
      "Requirement already satisfied: tensorflow-estimator<2.12,>=2.11.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (2.11.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.4.0)\n",
      "Requirement already satisfied: keras<2.12,>=2.11.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (2.11.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (3.7.4.3)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.9/site-packages (from tensorflow) (59.8.0)\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.15.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (3.19.3)\n",
      "Requirement already satisfied: numpy>=1.20 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (1.24.2)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /opt/conda/lib/python3.9/site-packages (from tensorflow) (0.31.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.9/site-packages (from astunparse>=1.6.0->tensorflow) (0.37.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.12,>=2.11->tensorflow) (0.6.1)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.12,>=2.11->tensorflow) (1.35.0)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.12,>=2.11->tensorflow) (2.27.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.12,>=2.11->tensorflow) (0.4.6)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.12,>=2.11->tensorflow) (1.8.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.12,>=2.11->tensorflow) (2.0.2)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.9/site-packages (from tensorboard<2.12,>=2.11->tensorflow) (3.3.6)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.9/site-packages (from packaging->tensorflow) (3.0.7)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow) (0.2.8)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow) (4.2.4)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow) (4.8)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.9/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.9/site-packages (from markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow) (4.10.1)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow) (2.0.10)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow) (1.26.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow) (2021.10.8)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.9/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow) (3.7.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.9/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow) (3.2.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow\n",
    "#!pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dbd010dd-8f29-48b1-9e89-2cab82e6ea60",
   "metadata": {
    "executionInfo": {
     "elapsed": 1080,
     "status": "ok",
     "timestamp": 1675097620466,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "dbd010dd-8f29-48b1-9e89-2cab82e6ea60"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "85187dff-a49f-4dab-acc5-0479a7694cb8",
   "metadata": {
    "executionInfo": {
     "elapsed": 682,
     "status": "ok",
     "timestamp": 1675097621145,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "85187dff-a49f-4dab-acc5-0479a7694cb8"
   },
   "outputs": [],
   "source": [
    "treino=pd.read_csv(data_path+'/comunic_treino_aug.csv', encoding='utf-8')\n",
    "teste=pd.read_csv(data_path+'/comunic_teste.csv', encoding='utf-8')\n",
    "validacao=pd.read_csv(data_path+'/comunic_valid.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "gFFnKgbUdaY0",
   "metadata": {
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1675097621146,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "gFFnKgbUdaY0"
   },
   "outputs": [],
   "source": [
    "def TFIDF(X_train, X_test,MAX_NB_WORDS=4000):\n",
    "    vectorizer_x = TfidfVectorizer(max_features=MAX_NB_WORDS)\n",
    "    X_train = vectorizer_x.fit_transform(X_train).toarray()\n",
    "    X_test = vectorizer_x.transform(X_test).toarray()\n",
    "    print(\"tf-idf with\",str(np.array(X_train).shape[1]),\"features\")\n",
    "    return (X_train,X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "734d7b64-95e4-4fc0-840f-f4f66e0ce340",
   "metadata": {
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1675097621146,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "734d7b64-95e4-4fc0-840f-f4f66e0ce340"
   },
   "outputs": [],
   "source": [
    "def make_tfidf(coluna):\n",
    "    vectorizer = TfidfVectorizer()\n",
    "    vectors = np.asarray(vectorizer.fit_transform(coluna).todense())\n",
    "    return vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "01f1e27a-ace3-48f6-a348-9466c2acfbd4",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1675097621147,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "01f1e27a-ace3-48f6-a348-9466c2acfbd4",
    "outputId": "ca87d1ac-ab75-43b8-b478-0c9a7f521efd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf-idf with 4000 features\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = TFIDF(treino[\"text\"], teste[\"text\"])\n",
    "Y_train=treino.label\n",
    "Y_test=teste.label\n",
    "X_val=make_tfidf(validacao.text)\n",
    "Y_val=validacao.label"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6932765d-6b4a-45bf-97c3-c7751a017724",
   "metadata": {
    "id": "6932765d-6b4a-45bf-97c3-c7751a017724"
   },
   "source": [
    "# Definição do modelo de classificacao DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "PuKm4srVFMLL",
   "metadata": {
    "executionInfo": {
     "elapsed": 2489,
     "status": "ok",
     "timestamp": 1675097623630,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "PuKm4srVFMLL"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-08 11:46:09.664488: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-03-08 11:46:10.219046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n",
      "2023-03-08 11:46:10.219096: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n",
      "2023-03-08 11:46:10.219103: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.keras import models\n",
    "from tensorflow.python.keras import layers\n",
    "from tensorflow.python.keras import losses\n",
    "from tensorflow.python.keras import optimizers\n",
    "\n",
    "from tensorflow.python.keras.callbacks import EarlyStopping\n",
    "\n",
    "early_stopping =  EarlyStopping(\n",
    "    monitor='val_accuracy', \n",
    "    patience=4\n",
    ")\n",
    "\n",
    "def Build_Model_DNN_Text(shape, nClasses, dropout=0.4):\n",
    "    model = models.Sequential()\n",
    "    model.add(layers.Dense(4000,input_dim=shape,activation='relu'))\n",
    "    model.add(layers.Dense(shape,activation='relu'))\n",
    "    model.add(layers.Dropout(dropout))\n",
    "    model.add(layers.Dense(1024,activation='relu'))\n",
    "    model.add(layers.Dropout(dropout))\n",
    "    model.add(layers.Dense(128,activation='relu'))\n",
    "    model.add(layers.Dropout(dropout))    \n",
    "    model.add(layers.Dense(1024,activation='relu'))\n",
    "    model.add(layers.Dropout(dropout))    \n",
    "    model.add(layers.Dense(nClasses, activation='sigmoid'))\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                  optimizer='Adam',\n",
    "                  metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "Gc8dcyskKuJ3",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 132261,
     "status": "ok",
     "timestamp": 1675097767282,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "Gc8dcyskKuJ3",
    "outputId": "d794d2cd-52ac-4e6d-9c34-6e523e1163f9"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-08 11:46:10.725291: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-08 11:46:10.733560: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-08 11:46:10.733807: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-08 11:46:10.734446: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-03-08 11:46:10.735142: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-08 11:46:10.735363: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-08 11:46:10.735562: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-08 11:46:11.077306: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-08 11:46:11.077496: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-08 11:46:11.077611: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-03-08 11:46:11.077704: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1613] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 5366 MB memory:  -> device: 0, name: Quadro RTX 4000, pci bus id: 0000:01:00.0, compute capability: 7.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 4000)              16004000  \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 4000)              16004000  \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 4000)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1024)              4097024   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 128)               131200    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1024)              132096    \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 13)                13325     \n",
      "=================================================================\n",
      "Total params: 36,381,645\n",
      "Trainable params: 36,381,645\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/50\n",
      "23/23 - 1s - loss: 2.3352 - accuracy: 0.1774 - val_loss: 1.5100 - val_accuracy: 0.4856\n",
      "Epoch 2/50\n",
      "23/23 - 0s - loss: 1.7817 - accuracy: 0.3571 - val_loss: 1.3215 - val_accuracy: 0.6049\n",
      "Epoch 3/50\n",
      "23/23 - 0s - loss: 0.7993 - accuracy: 0.6941 - val_loss: 1.2843 - val_accuracy: 0.7325\n",
      "Epoch 4/50\n",
      "23/23 - 0s - loss: 0.2136 - accuracy: 0.9311 - val_loss: 1.3951 - val_accuracy: 0.7737\n",
      "Epoch 5/50\n",
      "23/23 - 0s - loss: 0.0371 - accuracy: 0.9901 - val_loss: 1.7727 - val_accuracy: 0.7901\n",
      "Epoch 6/50\n",
      "23/23 - 0s - loss: 0.0283 - accuracy: 0.9935 - val_loss: 1.8713 - val_accuracy: 0.7819\n",
      "Epoch 7/50\n",
      "23/23 - 0s - loss: 0.0186 - accuracy: 0.9969 - val_loss: 1.7174 - val_accuracy: 0.7819\n",
      "Epoch 8/50\n",
      "23/23 - 0s - loss: 0.0099 - accuracy: 0.9976 - val_loss: 1.9111 - val_accuracy: 0.7860\n",
      "Epoch 9/50\n",
      "23/23 - 0s - loss: 0.0083 - accuracy: 0.9990 - val_loss: 1.9033 - val_accuracy: 0.7737\n",
      "Epoch 10/50\n",
      "23/23 - 0s - loss: 0.0067 - accuracy: 0.9983 - val_loss: 2.0515 - val_accuracy: 0.7778\n",
      "Epoch 11/50\n",
      "23/23 - 0s - loss: 0.0050 - accuracy: 0.9983 - val_loss: 1.9520 - val_accuracy: 0.7860\n",
      "Epoch 12/50\n",
      "23/23 - 0s - loss: 0.0059 - accuracy: 0.9986 - val_loss: 2.5488 - val_accuracy: 0.7778\n",
      "Epoch 13/50\n",
      "23/23 - 0s - loss: 0.0035 - accuracy: 0.9993 - val_loss: 2.0373 - val_accuracy: 0.7984\n",
      "Epoch 14/50\n",
      "23/23 - 0s - loss: 0.0033 - accuracy: 0.9993 - val_loss: 2.1961 - val_accuracy: 0.7819\n",
      "Epoch 15/50\n",
      "23/23 - 0s - loss: 0.0038 - accuracy: 0.9993 - val_loss: 2.1319 - val_accuracy: 0.7901\n",
      "Epoch 16/50\n",
      "23/23 - 0s - loss: 0.0048 - accuracy: 0.9990 - val_loss: 2.4362 - val_accuracy: 0.7654\n",
      "Epoch 17/50\n",
      "23/23 - 0s - loss: 0.0087 - accuracy: 0.9976 - val_loss: 2.3560 - val_accuracy: 0.7819\n",
      "Epoch 18/50\n",
      "23/23 - 0s - loss: 0.0043 - accuracy: 0.9990 - val_loss: 2.5026 - val_accuracy: 0.7942\n",
      "Epoch 19/50\n",
      "23/23 - 0s - loss: 0.0081 - accuracy: 0.9976 - val_loss: 2.5060 - val_accuracy: 0.7901\n",
      "Epoch 20/50\n",
      "23/23 - 0s - loss: 0.0278 - accuracy: 0.9949 - val_loss: 2.4917 - val_accuracy: 0.7695\n",
      "Epoch 21/50\n",
      "23/23 - 0s - loss: 0.0169 - accuracy: 0.9966 - val_loss: 3.0122 - val_accuracy: 0.7778\n",
      "Epoch 22/50\n",
      "23/23 - 0s - loss: 0.0202 - accuracy: 0.9959 - val_loss: 2.5712 - val_accuracy: 0.7654\n",
      "Epoch 23/50\n",
      "23/23 - 0s - loss: 0.0283 - accuracy: 0.9945 - val_loss: 2.3863 - val_accuracy: 0.7325\n",
      "Epoch 24/50\n",
      "23/23 - 0s - loss: 0.0457 - accuracy: 0.9898 - val_loss: 2.4572 - val_accuracy: 0.7490\n",
      "Epoch 25/50\n",
      "23/23 - 0s - loss: 0.0313 - accuracy: 0.9942 - val_loss: 2.7880 - val_accuracy: 0.7366\n",
      "Epoch 26/50\n",
      "23/23 - 0s - loss: 0.0097 - accuracy: 0.9966 - val_loss: 2.5104 - val_accuracy: 0.7654\n",
      "Epoch 27/50\n",
      "23/23 - 0s - loss: 0.0251 - accuracy: 0.9932 - val_loss: 2.4529 - val_accuracy: 0.7778\n",
      "Epoch 28/50\n",
      "23/23 - 0s - loss: 0.0238 - accuracy: 0.9956 - val_loss: 1.9334 - val_accuracy: 0.7695\n",
      "Epoch 29/50\n",
      "23/23 - 0s - loss: 0.0094 - accuracy: 0.9976 - val_loss: 2.6054 - val_accuracy: 0.7778\n",
      "Epoch 30/50\n",
      "23/23 - 0s - loss: 0.0102 - accuracy: 0.9993 - val_loss: 2.5864 - val_accuracy: 0.7778\n",
      "Epoch 31/50\n",
      "23/23 - 0s - loss: 0.0057 - accuracy: 0.9976 - val_loss: 2.5808 - val_accuracy: 0.7819\n",
      "Epoch 32/50\n",
      "23/23 - 0s - loss: 0.0311 - accuracy: 0.9980 - val_loss: 2.1590 - val_accuracy: 0.7860\n",
      "Epoch 33/50\n",
      "23/23 - 0s - loss: 0.0063 - accuracy: 0.9986 - val_loss: 2.3217 - val_accuracy: 0.7942\n",
      "Epoch 34/50\n",
      "23/23 - 0s - loss: 0.0047 - accuracy: 0.9990 - val_loss: 2.5445 - val_accuracy: 0.7901\n",
      "Epoch 35/50\n",
      "23/23 - 0s - loss: 0.0061 - accuracy: 0.9986 - val_loss: 2.7326 - val_accuracy: 0.7778\n",
      "Epoch 36/50\n",
      "23/23 - 0s - loss: 0.0037 - accuracy: 0.9993 - val_loss: 2.7447 - val_accuracy: 0.7778\n",
      "Epoch 37/50\n",
      "23/23 - 0s - loss: 0.0025 - accuracy: 0.9993 - val_loss: 2.7692 - val_accuracy: 0.7819\n",
      "Epoch 38/50\n",
      "23/23 - 0s - loss: 0.0042 - accuracy: 0.9997 - val_loss: 2.8350 - val_accuracy: 0.7860\n",
      "Epoch 39/50\n",
      "23/23 - 0s - loss: 0.0052 - accuracy: 0.9990 - val_loss: 2.6940 - val_accuracy: 0.7942\n",
      "Epoch 40/50\n",
      "23/23 - 0s - loss: 0.0031 - accuracy: 0.9993 - val_loss: 2.7891 - val_accuracy: 0.7860\n",
      "Epoch 41/50\n",
      "23/23 - 0s - loss: 0.0028 - accuracy: 0.9993 - val_loss: 2.7796 - val_accuracy: 0.7860\n",
      "Epoch 42/50\n",
      "23/23 - 0s - loss: 0.0090 - accuracy: 0.9986 - val_loss: 2.8737 - val_accuracy: 0.7778\n",
      "Epoch 43/50\n",
      "23/23 - 0s - loss: 0.0015 - accuracy: 0.9997 - val_loss: 2.7604 - val_accuracy: 0.7860\n",
      "Epoch 44/50\n",
      "23/23 - 0s - loss: 0.0018 - accuracy: 0.9997 - val_loss: 2.7625 - val_accuracy: 0.7901\n",
      "Epoch 45/50\n",
      "23/23 - 0s - loss: 0.0019 - accuracy: 0.9997 - val_loss: 2.8023 - val_accuracy: 0.7901\n",
      "Epoch 46/50\n",
      "23/23 - 0s - loss: 0.0019 - accuracy: 0.9997 - val_loss: 2.7977 - val_accuracy: 0.7942\n",
      "Epoch 47/50\n",
      "23/23 - 0s - loss: 9.1744e-04 - accuracy: 0.9997 - val_loss: 2.8775 - val_accuracy: 0.7860\n",
      "Epoch 48/50\n",
      "23/23 - 0s - loss: 0.0014 - accuracy: 0.9997 - val_loss: 2.8837 - val_accuracy: 0.7901\n",
      "Epoch 49/50\n",
      "23/23 - 0s - loss: 0.0013 - accuracy: 0.9997 - val_loss: 2.9085 - val_accuracy: 0.7901\n",
      "Epoch 50/50\n",
      "23/23 - 0s - loss: 0.0013 - accuracy: 0.9997 - val_loss: 2.9435 - val_accuracy: 0.7901\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.93      0.74        15\n",
      "           1       0.82      0.78      0.80        18\n",
      "           2       0.33      0.33      0.33         3\n",
      "           3       0.33      0.29      0.31         7\n",
      "           4       0.91      0.97      0.94       114\n",
      "           5       0.67      0.89      0.76         9\n",
      "           6       1.00      0.50      0.67         4\n",
      "           7       0.62      0.36      0.45        14\n",
      "           8       0.50      0.50      0.50         4\n",
      "           9       0.57      0.59      0.58        22\n",
      "          10       1.00      0.29      0.44         7\n",
      "          11       1.00      0.25      0.40         4\n",
      "          12       0.80      0.73      0.76        22\n",
      "\n",
      "    accuracy                           0.79       243\n",
      "   macro avg       0.71      0.57      0.59       243\n",
      "weighted avg       0.79      0.79      0.77       243\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "model_DNN = Build_Model_DNN_Text(X_train.shape[1], 13)\n",
    "model_DNN.fit(X_train, Y_train,\n",
    "                              validation_data=(X_test, Y_test),\n",
    "                              epochs=50,\n",
    "                              batch_size=128,\n",
    "                              verbose=2)\n",
    "                              #class_weight=class_weights,\n",
    "                              #callbacks=[early_stopping])\n",
    "predicted = model_DNN.predict(X_test)\n",
    "Y_pred = np.argmax(predicted, axis=1)\n",
    "print(metrics.classification_report(Y_test, Y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3b1b126-a5c7-4caf-8a94-77ea31b690f1",
   "metadata": {
    "id": "c3b1b126-a5c7-4caf-8a94-77ea31b690f1"
   },
   "source": [
    "### Métricas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3810f4d0-4f8f-424e-b71c-9cd3fb09e71e",
   "metadata": {
    "executionInfo": {
     "elapsed": 387,
     "status": "ok",
     "timestamp": 1675097825090,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "3810f4d0-4f8f-424e-b71c-9cd3fb09e71e"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score, roc_auc_score, accuracy_score, precision_score, recall_score, hamming_loss\n",
    "precision = precision_score(Y_test, Y_pred,average='micro')\n",
    "recall = recall_score(Y_test, Y_pred,average='micro')\n",
    "f1_micro_average = f1_score(y_true=Y_test, y_pred=Y_pred, average='micro')\n",
    "#roc_auc = roc_auc_score(Y_test, Y_pred, multi_class ='ovr', average = 'micro')\n",
    "accuracy = accuracy_score(Y_test, Y_pred)\n",
    "hloss = hamming_loss(Y_test, Y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "503cc683",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 433,
     "status": "ok",
     "timestamp": 1675097833896,
     "user": {
      "displayName": "Fábio Cordeiro",
      "userId": "00933077666862327455"
     },
     "user_tz": 180
    },
    "id": "503cc683",
    "outputId": "6720b7f7-483b-4245-fa41-102ed3627d6c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acurácia: 0.7860082304526749\n",
      "Precisão: 0.7860082304526749\n",
      "Recall: 0.7860082304526749\n",
      "F1: 0.7860082304526749\n",
      "Hamming Loss: 0.2139917695473251\n"
     ]
    }
   ],
   "source": [
    "print(\"Acurácia:\",accuracy)\n",
    "print(\"Precisão:\",precision)\n",
    "print(\"Recall:\",recall)\n",
    "print(\"F1:\",f1_micro_average)\n",
    "#print(\"Roc_auc:\",roc_auc)\n",
    "print(\"Hamming Loss:\",hloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4871accb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}