GNN_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import dgl\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making the comb files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Eval combinations\n",
    "eval_df = pd.read_csv('../data/TWOSIDE-evaluation-PSE-964.csv', sep=',')\n",
    "eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drugs_list = eval_df['Drug1'].unique().tolist()\n",
    "for drug in eval_df['Drug2'].unique().tolist():\n",
    "    if drug not in drugs_list:\n",
    "        drugs_list.append(drug)\n",
    "\n",
    "len(drugs_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from urllib.request import urlopen\n",
    "\n",
    "f = open('../data/Eval_drugs_964.tsv', 'a')\n",
    "f.write('Drug_name\\tPubChemID\\tSMILES\\n')\n",
    "\n",
    "for drug in tqdm(drugs_list):\n",
    "    drug_name = drug\n",
    "    \n",
    "    if ' ' in drug:\n",
    "        drug = drug.replace(' ', '%20')\n",
    "        \n",
    "    try:\n",
    "        url1 = 'https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/'+drug+'/property/CanonicalSMILES/TXT'\n",
    "        url2 = 'https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/'+drug+'/cids/TXT'\n",
    "        res1 = urlopen(url1)\n",
    "        smiles = str(res1.read())[2:-3]\n",
    "        res2 = urlopen(url2)\n",
    "        drugid = str(res2.read())[2:-3].split('\\\\n')[0]\n",
    "        \n",
    "        row = drug_name + '\\t' + drugid +'\\t'+ smiles + '\\n'\n",
    "        f.write(row)\n",
    "    except:\n",
    "        #row = drug_name + '\\t' + '-\\n'\n",
    "        #f.write(row)\n",
    "        pass\n",
    "\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('../data/Eval_drugs_964.tsv', sep='\\t')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making the DTI and DrugID file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DrugID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DrugID\n",
    "eval_drugs = pd.read_csv('../data/Eval_drugs_964.tsv', sep='\\t')\n",
    "eval_drugs = eval_drugs[['Drug_name','PubChemID']]\n",
    "eval_drugs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_drugs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drugs = eval_drugs.sort_values(by='PubChemID')['PubChemID'].unique().tolist()\n",
    "drugs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_drugs['GraphID'] = '-'\n",
    "drugs = eval_drugs.sort_values(by='PubChemID')['PubChemID'].unique().tolist()\n",
    "dic = {drug:drugs.index(drug)+1 for drug in drugs} # conversion dic, starts at 0\n",
    "eval_drugs['GraphID'] = eval_drugs['PubChemID'].map(dic) #DrugIDs\n",
    "eval_drugs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_drugs = eval_drugs.sort_values(by='GraphID')\n",
    "eval_drugs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_drugs = eval_drugs.rename({'Drug_name':'Name','PubChemID':'DrugID'}, axis=1)\n",
    "eval_drugs = eval_drugs[['GraphID','DrugID','Name']]\n",
    "eval_drugs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_drugs.to_csv('../data/Eval_DrugID.csv', index=False, sep = ',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DTI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_dti = pd.read_csv('../data/Eval_affinity_cut_83.37.csv', sep=',')\n",
    "eval_dti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_id = pd.read_csv('../data/GeneID.csv', sep=',')\n",
    "gene_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_dti['ProteinID'] = '-'\n",
    "genes = gene_id['Name'].tolist()\n",
    "gene_dic = {gene:genes.index(gene)+1 for gene in genes}\n",
    "eval_dti['ProteinID'] = eval_dti['GeneID'].map(gene_dic) #DrugIDs\n",
    "eval_dti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_dti[['DrugID','ProteinID']].to_csv('../data/Eval_DTI_full.csv', index=False, sep = ',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import dgl\n",
    "import time\n",
    "import torch\n",
    "import pandas as pd\n",
    "import torch.nn as nn\n",
    "from tqdm.notebook import tqdm\n",
    "from sklearn import metrics\n",
    "import torch.nn.functional as F\n",
    "from dgl.data import DGLDataset\n",
    "torch.cuda.set_device(0)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Creating the Evaluation Dataset ...\n",
      "\n",
      "data loaded!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2111b0da43154e03a12977a561bf03a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Evaluation Dataset created!\n",
      "\n",
      "\n",
      "dataset is compiled! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "class PSE_eval(DGLDataset):\n",
    "    def __init__(self):\n",
    "        super().__init__(name='PSE_eval')\n",
    "\n",
    "    def process(self):\n",
    "        features = pd.read_csv('../data/GNN-GSE_full_pkd_norm.csv',index_col = 'ProteinID', sep=',')\n",
    "        drug_comb = pd.read_csv('../data/Eval_TWOSIDE-evaluation-PSE-964.csv', sep=',') \n",
    "        nodes = pd.read_csv('../data/GNN-GSE_full_pkd_norm.csv', sep=',')\n",
    "        edges = pd.read_csv('../data/GNN-PPI-net.csv', sep=',')\n",
    "        dti = pd.read_csv('../data/Eval_DTI_full.csv', sep=',')\n",
    "        DrugID = pd.read_csv('../data/Eval_DrugID.csv', sep = ',')\n",
    "        print('data loaded!')\n",
    "        \n",
    "        # generate drug specific ppi subgraph for GNN edges\n",
    "        def drug2ppi(drug):\n",
    "            genes = dti['ProteinID'].loc[dti['DrugID'] == drug].tolist()\n",
    "            df = edges[['protein1','protein2']].loc[edges['protein1'].isin(genes)]\n",
    "            df = df.loc[df['protein2'].isin(genes)]\n",
    "            num_nodes = len(df['protein1'].unique())\n",
    "            df['graph_id'] = DrugID.loc[DrugID['DrugID'] == drug]['GraphID'].tolist()[0]  #DrugID\n",
    "            df = df.rename(columns={'protein1': 'src_prot', 'protein2': 'dst_prot'}) # prot: actual protein id\n",
    "            final_genes =df['src_prot'].unique().tolist() # final genes that have ppi data\n",
    "            dic = {gene:final_genes.index(gene) for gene in final_genes} # conversion dic, starts at 0\n",
    "            df['src'] = df['src_prot'].map(dic) #local ids\n",
    "            df['dst'] = df['dst_prot'].map(dic) #local ids\n",
    "            return(df[['graph_id', 'src', 'dst', 'src_prot', 'dst_prot']],num_nodes)\n",
    "        \n",
    "        self.graphs = []\n",
    "        self.labels = []\n",
    "        self.comb_graphs = []\n",
    "        self.comb_labels = []\n",
    "\n",
    "        #Node features or PSEs dictionary\n",
    "        feature_dic = {i+1:torch.tensor(features.loc[i+1,]) for i in range(len(features))}\n",
    "    \n",
    "        # For each graph ID...\n",
    "        for drug in tqdm(DrugID['DrugID'].tolist()[:100]):\n",
    "            # Find the edges as well as the number of nodes and its label.\n",
    "            edges_of_id,num_nodes = drug2ppi(drug)\n",
    "            src = edges_of_id['src'].to_numpy()\n",
    "            dst = edges_of_id['dst'].to_numpy()\n",
    "            label = DrugID.loc[DrugID['DrugID'] == drug]['Name'].tolist()[0]\n",
    "            \n",
    "            # Create a graph and add it to the list of graphs and labels.\n",
    "            g = dgl.graph((src, dst), num_nodes=num_nodes)\n",
    "            \n",
    "            # Need to convert proteinsIDs for feature assigning\n",
    "            prot_ids = edges_of_id['src_prot'].unique().tolist()\n",
    "            for prot in edges_of_id['dst_prot'].unique().tolist():\n",
    "                if prot not in prot_ids:\n",
    "                    prot_ids.append(prot)\n",
    "            convert_prot = {prot_ids.index(prot):prot for prot in prot_ids}\n",
    "            \n",
    "            #Adding features of each node\n",
    "            g.ndata['PSE'] = torch.zeros(g.num_nodes(), 964)\n",
    "            for node in g.nodes().tolist():\n",
    "                g.ndata['PSE'][node] = feature_dic[convert_prot[node]]\n",
    "                \n",
    "            self.graphs.append(g)\n",
    "            self.labels.append(label)\n",
    "            \n",
    "        # conver drugid to their respective graph id\n",
    "        #drug2graph = {properties['label'][i]:i for i in range(len(properties))} \n",
    "        #drug2graph = {self.labels[i]:i for i in range(len(self.labels))} \n",
    "        \n",
    "        for i in range(len(drug_comb)):\n",
    "            row = drug_comb.loc[i]\n",
    "            try:\n",
    "                g1 = self.graphs[self.labels.index(row[0])] # Drug1 graph\n",
    "                g2 = self.graphs[self.labels.index(row[1])] # Drug2 graph  \n",
    "                self.comb_graphs.append([g1,g2])\n",
    "                self.comb_labels.append(torch.tensor(row[2:])) # PSE values\n",
    "            except:\n",
    "                pass\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.comb_graphs[i], self.comb_labels[i]\n",
    "        #return self.graphs[i], self.labels[i]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.comb_graphs)\n",
    "    \n",
    "print('\\nCreating the Evaluation Dataset ...\\n')\n",
    "dataset = PSE_eval()\n",
    "\n",
    "print('\\nEvaluation Dataset created!\\n')\n",
    "\n",
    "print('\\ndataset is compiled! \\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Creating eval batches ...\n",
      "\n",
      "\n",
      "Eval batches are created!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('\\nCreating eval batches ...\\n')\n",
    "# Making the batches\n",
    "from dgl.dataloading import GraphDataLoader\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "\n",
    "num_examples = len(dataset)\n",
    "eval_sampler = SubsetRandomSampler(torch.arange(num_examples))\n",
    "eval_dataloader = GraphDataLoader(dataset, sampler=eval_sampler, batch_size=1, drop_last=False)\n",
    "\n",
    "print('\\nEval batches are created!\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GNN Model: Siamese GCN \n",
    "from dgl.nn import GraphConv\n",
    "\n",
    "class GCN(nn.Module):\n",
    "    def __init__(self, in_feats, h_feats, num_classes):\n",
    "        super(GCN, self).__init__()\n",
    "        self.conv1 = GraphConv(in_feats, h_feats)\n",
    "        self.conv2 = GraphConv(h_feats,  num_classes)\n",
    "        \n",
    "    def forward(self, g, in_feat):\n",
    "        h = self.conv1(g, in_feat)\n",
    "        h = F.relu(h)\n",
    "        h = self.conv2(g, h)\n",
    "        g.ndata['h'] = h\n",
    "        out = F.relu(dgl.mean_nodes(g, 'h'))\n",
    "        #out = F.relu(dgl.max_nodes(g, 'h'))\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating \n",
      "\n",
      "TP:27,TN:853,FP:7,FN:77\n",
      "Accuracy: 91.2863 | Precision: 79.4118 | Recall: 25.9615 | F1: 39.1304 | Specificity: 99.186 | MCC: 0.7115\n",
      "\n",
      "TP:2,TN:931,FP:28,FN:3\n",
      "Accuracy: 96.7842 | Precision: 6.6667 | Recall: 40.0 | F1: 11.4286 | Specificity: 97.0803 | MCC: 0.5767\n",
      "\n",
      "TP:10,TN:912,FP:24,FN:18\n",
      "Accuracy: 95.6432 | Precision: 29.4118 | Recall: 35.7143 | F1: 32.2581 | Specificity: 97.4359 | MCC: 0.6509\n",
      "\n",
      "TP:13,TN:904,FP:17,FN:30\n",
      "Accuracy: 95.1245 | Precision: 43.3333 | Recall: 30.2326 | F1: 35.6164 | Specificity: 98.1542 | MCC: 0.6687\n",
      "\n",
      "TP:21,TN:882,FP:13,FN:48\n",
      "Accuracy: 93.6722 | Precision: 61.7647 | Recall: 30.4348 | F1: 40.7767 | Specificity: 98.5475 | MCC: 0.7025\n",
      "\n",
      "TP:24,TN:859,FP:6,FN:75\n",
      "Accuracy: 91.5975 | Precision: 80.0 | Recall: 24.2424 | F1: 37.2093 | Specificity: 99.3064 | MCC: 0.7058\n",
      "\n",
      "TP:19,TN:875,FP:15,FN:55\n",
      "Accuracy: 92.7386 | Precision: 55.8824 | Recall: 25.6757 | F1: 35.1852 | Specificity: 98.3146 | MCC: 0.6731\n",
      "\n",
      "TP:11,TN:919,FP:23,FN:11\n",
      "Accuracy: 96.473 | Precision: 32.3529 | Recall: 50.0 | F1: 39.2857 | Specificity: 97.5584 | MCC: 0.6925\n",
      "\n",
      "TP:17,TN:881,FP:17,FN:49\n",
      "Accuracy: 93.1535 | Precision: 50.0 | Recall: 25.7576 | F1: 34.0 | Specificity: 98.1069 | MCC: 0.6634\n",
      "\n",
      "TP:5,TN:920,FP:29,FN:10\n",
      "Accuracy: 95.9544 | Precision: 14.7059 | Recall: 33.3333 | F1: 20.4082 | Specificity: 96.9442 | MCC: 0.6016\n",
      "\n",
      "TP:20,TN:888,FP:14,FN:42\n",
      "Accuracy: 94.1909 | Precision: 58.8235 | Recall: 32.2581 | F1: 41.6667 | Specificity: 98.4479 | MCC: 0.7042\n",
      "\n",
      "TP:24,TN:865,FP:10,FN:65\n",
      "Accuracy: 92.2199 | Precision: 70.5882 | Recall: 26.9663 | F1: 39.0244 | Specificity: 98.8571 | MCC: 0.7026\n",
      "\n",
      "TP:13,TN:894,FP:21,FN:36\n",
      "Accuracy: 94.0871 | Precision: 38.2353 | Recall: 26.5306 | F1: 31.3253 | Specificity: 97.7049 | MCC: 0.6443\n",
      "\n",
      "TP:7,TN:918,FP:27,FN:12\n",
      "Accuracy: 95.9544 | Precision: 20.5882 | Recall: 36.8421 | F1: 26.4151 | Specificity: 97.1429 | MCC: 0.628\n",
      "\n",
      "TP:19,TN:854,FP:11,FN:80\n",
      "Accuracy: 90.5602 | Precision: 63.3333 | Recall: 19.1919 | F1: 29.4574 | Specificity: 98.7283 | MCC: 0.6566\n",
      "\n",
      "TP:17,TN:906,FP:17,FN:24\n",
      "Accuracy: 95.7469 | Precision: 50.0 | Recall: 41.4634 | F1: 45.3333 | Specificity: 98.1582 | MCC: 0.7167\n",
      "\n",
      "TP:7,TN:909,FP:27,FN:21\n",
      "Accuracy: 95.0207 | Precision: 20.5882 | Recall: 25.0 | F1: 22.5806 | Specificity: 97.1154 | MCC: 0.6007\n",
      "\n",
      "TP:7,TN:927,FP:27,FN:3\n",
      "Accuracy: 96.888 | Precision: 20.5882 | Recall: 70.0 | F1: 31.8182 | Specificity: 97.1698 | MCC: 0.6845\n",
      "\n",
      "TP:20,TN:870,FP:14,FN:60\n",
      "Accuracy: 92.3237 | Precision: 58.8235 | Recall: 25.0 | F1: 35.0877 | Specificity: 98.4163 | MCC: 0.6751\n",
      "\n",
      "TP:9,TN:895,FP:25,FN:35\n",
      "Accuracy: 93.7759 | Precision: 26.4706 | Recall: 20.4545 | F1: 23.0769 | Specificity: 97.2826 | MCC: 0.6003\n",
      "\n",
      "TP:21,TN:844,FP:9,FN:90\n",
      "Accuracy: 89.7303 | Precision: 70.0 | Recall: 18.9189 | F1: 29.7872 | Specificity: 98.9449 | MCC: 0.6642\n",
      "\n",
      "TP:3,TN:921,FP:31,FN:9\n",
      "Accuracy: 95.8506 | Precision: 8.8235 | Recall: 25.0 | F1: 13.0435 | Specificity: 96.7437 | MCC: 0.5653\n",
      "\n",
      "TP:22,TN:879,FP:12,FN:51\n",
      "Accuracy: 93.4647 | Precision: 64.7059 | Recall: 30.137 | F1: 41.1215 | Specificity: 98.6532 | MCC: 0.7065\n",
      "\n",
      "TP:5,TN:907,FP:29,FN:23\n",
      "Accuracy: 94.6058 | Precision: 14.7059 | Recall: 17.8571 | F1: 16.129 | Specificity: 96.9017 | MCC: 0.5672\n",
      "\n",
      "TP:17,TN:851,FP:17,FN:79\n",
      "Accuracy: 90.0415 | Precision: 50.0 | Recall: 17.7083 | F1: 26.1538 | Specificity: 98.0415 | MCC: 0.6278\n",
      "\n",
      "TP:28,TN:775,FP:6,FN:155\n",
      "Accuracy: 83.2988 | Precision: 82.3529 | Recall: 15.3005 | F1: 25.8065 | Specificity: 99.2318 | MCC: 0.6545\n",
      "\n",
      "TP:16,TN:852,FP:18,FN:78\n",
      "Accuracy: 90.0415 | Precision: 47.0588 | Recall: 17.0213 | F1: 25.0 | Specificity: 97.931 | MCC: 0.6202\n",
      "\n",
      "TP:3,TN:931,FP:27,FN:3\n",
      "Accuracy: 96.888 | Precision: 10.0 | Recall: 50.0 | F1: 16.6667 | Specificity: 97.1816 | MCC: 0.6068\n",
      "\n",
      "TP:28,TN:802,FP:6,FN:128\n",
      "Accuracy: 86.0996 | Precision: 82.3529 | Recall: 17.9487 | F1: 29.4737 | Specificity: 99.2574 | MCC: 0.6718\n",
      "\n",
      "TP:21,TN:838,FP:13,FN:92\n",
      "Accuracy: 89.1079 | Precision: 61.7647 | Recall: 18.5841 | F1: 28.5714 | Specificity: 98.4724 | MCC: 0.6487\n",
      "\n",
      "TP:9,TN:924,FP:21,FN:10\n",
      "Accuracy: 96.7842 | Precision: 30.0 | Recall: 47.3684 | F1: 36.7347 | Specificity: 97.7778 | MCC: 0.6807\n",
      "\n",
      "TP:11,TN:887,FP:23,FN:43\n",
      "Accuracy: 93.1535 | Precision: 32.3529 | Recall: 20.3704 | F1: 25.0 | Specificity: 97.4725 | MCC: 0.6112\n",
      "\n",
      "TP:7,TN:916,FP:27,FN:14\n",
      "Accuracy: 95.7469 | Precision: 20.5882 | Recall: 33.3333 | F1: 25.4545 | Specificity: 97.1368 | MCC: 0.6206\n",
      "\n",
      "TP:16,TN:903,FP:18,FN:27\n",
      "Accuracy: 95.332 | Precision: 47.0588 | Recall: 37.2093 | F1: 41.5584 | Specificity: 98.0456 | MCC: 0.6973\n",
      "\n",
      "TP:22,TN:850,FP:12,FN:80\n",
      "Accuracy: 90.4564 | Precision: 64.7059 | Recall: 21.5686 | F1: 32.3529 | Specificity: 98.6079 | MCC: 0.6682\n",
      "\n",
      "TP:7,TN:900,FP:27,FN:30\n",
      "Accuracy: 94.0871 | Precision: 20.5882 | Recall: 18.9189 | F1: 19.7183 | Specificity: 97.0874 | MCC: 0.5834\n",
      "\n",
      "TP:11,TN:902,FP:23,FN:28\n",
      "Accuracy: 94.7095 | Precision: 32.3529 | Recall: 28.2051 | F1: 30.137 | Specificity: 97.5135 | MCC: 0.6374\n",
      "\n",
      "TP:20,TN:855,FP:10,FN:79\n",
      "Accuracy: 90.7676 | Precision: 66.6667 | Recall: 20.202 | F1: 31.0078 | Specificity: 98.8439 | MCC: 0.6665\n",
      "\n",
      "TP:8,TN:898,FP:26,FN:32\n",
      "Accuracy: 93.9834 | Precision: 23.5294 | Recall: 20.0 | F1: 21.6216 | Specificity: 97.1861 | MCC: 0.5929\n",
      "\n",
      "TP:20,TN:890,FP:14,FN:40\n",
      "Accuracy: 94.3983 | Precision: 58.8235 | Recall: 33.3333 | F1: 42.5532 | Specificity: 98.4513 | MCC: 0.7081\n",
      "\n",
      "TP:10,TN:917,FP:24,FN:13\n",
      "Accuracy: 96.1618 | Precision: 29.4118 | Recall: 43.4783 | F1: 35.0877 | Specificity: 97.4495 | MCC: 0.6693\n",
      "\n",
      "TP:10,TN:917,FP:24,FN:13\n",
      "Accuracy: 96.1618 | Precision: 29.4118 | Recall: 43.4783 | F1: 35.0877 | Specificity: 97.4495 | MCC: 0.6693\n",
      "\n",
      "TP:12,TN:881,FP:22,FN:49\n",
      "Accuracy: 92.6349 | Precision: 35.2941 | Recall: 19.6721 | F1: 25.2632 | Specificity: 97.5637 | MCC: 0.6137\n",
      "\n",
      "TP:17,TN:890,FP:17,FN:40\n",
      "Accuracy: 94.0871 | Precision: 50.0 | Recall: 29.8246 | F1: 37.3626 | Specificity: 98.1257 | MCC: 0.6787\n",
      "\n",
      "TP:14,TN:908,FP:20,FN:22\n",
      "Accuracy: 95.6432 | Precision: 41.1765 | Recall: 38.8889 | F1: 40.0 | Specificity: 97.8448 | MCC: 0.6888\n",
      "\n",
      "TP:2,TN:924,FP:32,FN:6\n",
      "Accuracy: 96.0581 | Precision: 5.8824 | Recall: 25.0 | F1: 9.5238 | Specificity: 96.6527 | MCC: 0.5532\n",
      "\n",
      "TP:21,TN:711,FP:9,FN:223\n",
      "Accuracy: 75.9336 | Precision: 70.0 | Recall: 8.6066 | F1: 15.3285 | Specificity: 98.75 | MCC: 0.5921\n",
      "\n",
      "TP:17,TN:893,FP:13,FN:41\n",
      "Accuracy: 94.3983 | Precision: 56.6667 | Recall: 29.3103 | F1: 38.6364 | Specificity: 98.5651 | MCC: 0.6909\n",
      "\n",
      "TP:13,TN:907,FP:17,FN:27\n",
      "Accuracy: 95.4357 | Precision: 43.3333 | Recall: 32.5 | F1: 37.1429 | Specificity: 98.1602 | MCC: 0.6761\n",
      "\n",
      "TP:2,TN:929,FP:32,FN:1\n",
      "Accuracy: 96.5768 | Precision: 5.8824 | Recall: 66.6667 | F1: 10.8108 | Specificity: 96.6701 | MCC: 0.5956\n",
      "\n",
      "TP:13,TN:898,FP:17,FN:36\n",
      "Accuracy: 94.5021 | Precision: 43.3333 | Recall: 26.5306 | F1: 32.9114 | Specificity: 98.1421 | MCC: 0.656\n",
      "\n",
      "TP:34,TN:96,FP:0,FN:834\n",
      "Accuracy: 13.4855 | Precision: 100.0 | Recall: 3.9171 | F1: 7.5388 | Specificity: 100.0 | MCC: 0.5318\n",
      "\n",
      "TP:14,TN:912,FP:20,FN:18\n",
      "Accuracy: 96.0581 | Precision: 41.1765 | Recall: 43.75 | F1: 42.4242 | Specificity: 97.8541 | MCC: 0.702\n",
      "\n",
      "TP:16,TN:896,FP:14,FN:38\n",
      "Accuracy: 94.6058 | Precision: 53.3333 | Recall: 29.6296 | F1: 38.0952 | Specificity: 98.4615 | MCC: 0.686\n",
      "\n",
      "TP:34,TN:590,FP:0,FN:340\n",
      "Accuracy: 64.7303 | Precision: 100.0 | Recall: 9.0909 | F1: 16.6667 | Specificity: 100.0 | MCC: 0.6201\n",
      "\n",
      "TP:15,TN:866,FP:19,FN:64\n",
      "Accuracy: 91.39 | Precision: 44.1176 | Recall: 18.9873 | F1: 26.5487 | Specificity: 97.8531 | MCC: 0.6252\n",
      "\n",
      "TP:9,TN:922,FP:25,FN:8\n",
      "Accuracy: 96.5768 | Precision: 26.4706 | Recall: 52.9412 | F1: 35.2941 | Specificity: 97.3601 | MCC: 0.6795\n",
      "\n",
      "TP:12,TN:903,FP:18,FN:31\n",
      "Accuracy: 94.917 | Precision: 40.0 | Recall: 27.907 | F1: 32.8767 | Specificity: 98.0456 | MCC: 0.6543\n",
      "\n",
      "TP:6,TN:922,FP:24,FN:12\n",
      "Accuracy: 96.2656 | Precision: 20.0 | Recall: 33.3333 | F1: 25.0 | Specificity: 97.463 | MCC: 0.62\n",
      "\n",
      "TP:19,TN:880,FP:15,FN:50\n",
      "Accuracy: 93.2573 | Precision: 55.8824 | Recall: 27.5362 | F1: 36.8932 | Specificity: 98.324 | MCC: 0.6807\n",
      "\n",
      "TP:4,TN:914,FP:30,FN:16\n",
      "Accuracy: 95.2282 | Precision: 11.7647 | Recall: 20.0 | F1: 14.8148 | Specificity: 96.822 | MCC: 0.565\n",
      "\n",
      "TP:11,TN:899,FP:23,FN:31\n",
      "Accuracy: 94.3983 | Precision: 32.3529 | Recall: 26.1905 | F1: 28.9474 | Specificity: 97.5054 | MCC: 0.6311\n",
      "\n",
      "TP:8,TN:919,FP:26,FN:11\n",
      "Accuracy: 96.1618 | Precision: 23.5294 | Recall: 42.1053 | F1: 30.1887 | Specificity: 97.2487 | MCC: 0.6483\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP:4,TN:919,FP:30,FN:11\n",
      "Accuracy: 95.7469 | Precision: 11.7647 | Recall: 26.6667 | F1: 16.3265 | Specificity: 96.8388 | MCC: 0.5789\n",
      "\n",
      "TP:22,TN:818,FP:12,FN:112\n",
      "Accuracy: 87.1369 | Precision: 64.7059 | Recall: 16.4179 | F1: 26.1905 | Specificity: 98.5542 | MCC: 0.6404\n",
      "\n",
      "TP:8,TN:908,FP:26,FN:22\n",
      "Accuracy: 95.0207 | Precision: 23.5294 | Recall: 26.6667 | F1: 25.0 | Specificity: 97.2163 | MCC: 0.6124\n",
      "\n",
      "TP:4,TN:921,FP:30,FN:9\n",
      "Accuracy: 95.9544 | Precision: 11.7647 | Recall: 30.7692 | F1: 17.0213 | Specificity: 96.8454 | MCC: 0.5863\n",
      "\n",
      "TP:10,TN:894,FP:24,FN:36\n",
      "Accuracy: 93.7759 | Precision: 29.4118 | Recall: 21.7391 | F1: 25.0 | Specificity: 97.3856 | MCC: 0.6105\n",
      "\n",
      "TP:24,TN:865,FP:10,FN:65\n",
      "Accuracy: 92.2199 | Precision: 70.5882 | Recall: 26.9663 | F1: 39.0244 | Specificity: 98.8571 | MCC: 0.7026\n",
      "\n",
      "TP:8,TN:910,FP:26,FN:20\n",
      "Accuracy: 95.2282 | Precision: 23.5294 | Recall: 28.5714 | F1: 25.8065 | Specificity: 97.2222 | MCC: 0.6174\n",
      "\n",
      "TP:8,TN:911,FP:26,FN:19\n",
      "Accuracy: 95.332 | Precision: 23.5294 | Recall: 29.6296 | F1: 26.2295 | Specificity: 97.2252 | MCC: 0.6201\n",
      "\n",
      "TP:13,TN:910,FP:21,FN:20\n",
      "Accuracy: 95.7469 | Precision: 38.2353 | Recall: 39.3939 | F1: 38.806 | Specificity: 97.7444 | MCC: 0.683\n",
      "\n",
      "TP:4,TN:932,FP:26,FN:2\n",
      "Accuracy: 97.0954 | Precision: 13.3333 | Recall: 66.6667 | F1: 22.2222 | Specificity: 97.286 | MCC: 0.6448\n",
      "\n",
      "TP:24,TN:754,FP:10,FN:176\n",
      "Accuracy: 80.7054 | Precision: 70.5882 | Recall: 12.0 | F1: 20.5128 | Specificity: 98.6911 | MCC: 0.6175\n",
      "\n",
      "TP:7,TN:874,FP:27,FN:56\n",
      "Accuracy: 91.39 | Precision: 20.5882 | Recall: 11.1111 | F1: 14.433 | Specificity: 97.0033 | MCC: 0.5544\n",
      "\n",
      "TP:9,TN:913,FP:25,FN:17\n",
      "Accuracy: 95.6432 | Precision: 26.4706 | Recall: 34.6154 | F1: 30.0 | Specificity: 97.3348 | MCC: 0.6403\n",
      "\n",
      "TP:27,TN:825,FP:7,FN:105\n",
      "Accuracy: 88.3817 | Precision: 79.4118 | Recall: 20.4545 | F1: 32.5301 | Specificity: 99.1587 | MCC: 0.6828\n",
      "\n",
      "TP:8,TN:913,FP:26,FN:17\n",
      "Accuracy: 95.5394 | Precision: 23.5294 | Recall: 32.0 | F1: 27.1186 | Specificity: 97.2311 | MCC: 0.6259\n",
      "\n",
      "TP:34,TN:352,FP:0,FN:578\n",
      "Accuracy: 40.0415 | Precision: 100.0 | Recall: 5.5556 | F1: 10.5263 | Specificity: 100.0 | MCC: 0.5725\n",
      "\n",
      "TP:7,TN:920,FP:27,FN:10\n",
      "Accuracy: 96.1618 | Precision: 20.5882 | Recall: 41.1765 | F1: 27.451 | Specificity: 97.1489 | MCC: 0.6367\n",
      "\n",
      "TP:12,TN:878,FP:22,FN:52\n",
      "Accuracy: 92.3237 | Precision: 35.2941 | Recall: 18.75 | F1: 24.4898 | Specificity: 97.5556 | MCC: 0.61\n",
      "\n",
      "TP:19,TN:887,FP:15,FN:43\n",
      "Accuracy: 93.9834 | Precision: 55.8824 | Recall: 30.6452 | F1: 39.5833 | Specificity: 98.337 | MCC: 0.6927\n",
      "\n",
      "TP:11,TN:906,FP:23,FN:24\n",
      "Accuracy: 95.1245 | Precision: 32.3529 | Recall: 31.4286 | F1: 31.8841 | Specificity: 97.5242 | MCC: 0.6468\n",
      "\n",
      "TP:34,TN:509,FP:0,FN:421\n",
      "Accuracy: 56.3278 | Precision: 100.0 | Recall: 7.4725 | F1: 13.9059 | Specificity: 100.0 | MCC: 0.6011\n",
      "\n",
      "TP:7,TN:913,FP:27,FN:17\n",
      "Accuracy: 95.4357 | Precision: 20.5882 | Recall: 29.1667 | F1: 24.1379 | Specificity: 97.1277 | MCC: 0.6111\n",
      "\n",
      "TP:21,TN:870,FP:13,FN:60\n",
      "Accuracy: 92.4274 | Precision: 61.7647 | Recall: 25.9259 | F1: 36.5217 | Specificity: 98.5277 | MCC: 0.6839\n",
      "\n",
      "TP:9,TN:920,FP:25,FN:10\n",
      "Accuracy: 96.3693 | Precision: 26.4706 | Recall: 47.3684 | F1: 33.9623 | Specificity: 97.3545 | MCC: 0.6685\n",
      "\n",
      "TP:4,TN:924,FP:30,FN:6\n",
      "Accuracy: 96.2656 | Precision: 11.7647 | Recall: 40.0 | F1: 18.1818 | Specificity: 96.8553 | MCC: 0.6012\n",
      "\n",
      "TP:7,TN:920,FP:27,FN:10\n",
      "Accuracy: 96.1618 | Precision: 20.5882 | Recall: 41.1765 | F1: 27.451 | Specificity: 97.1489 | MCC: 0.6367\n",
      "\n",
      "TP:7,TN:920,FP:27,FN:10\n",
      "Accuracy: 96.1618 | Precision: 20.5882 | Recall: 41.1765 | F1: 27.451 | Specificity: 97.1489 | MCC: 0.6367\n",
      "\n",
      "TP:2,TN:905,FP:29,FN:28\n",
      "Accuracy: 94.0871 | Precision: 6.4516 | Recall: 6.6667 | F1: 6.5574 | Specificity: 96.8951 | MCC: 0.5175\n",
      "\n",
      "TP:34,TN:466,FP:0,FN:464\n",
      "Accuracy: 51.8672 | Precision: 100.0 | Recall: 6.8273 | F1: 12.782 | Specificity: 100.0 | MCC: 0.5925\n",
      "\n",
      "TP:8,TN:931,FP:22,FN:3\n",
      "Accuracy: 97.4066 | Precision: 26.6667 | Recall: 72.7273 | F1: 39.0244 | Specificity: 97.6915 | MCC: 0.7154\n",
      "\n",
      "TP:9,TN:919,FP:25,FN:11\n",
      "Accuracy: 96.2656 | Precision: 26.4706 | Recall: 45.0 | F1: 33.3333 | Specificity: 97.3517 | MCC: 0.6636\n",
      "\n",
      "TP:15,TN:858,FP:19,FN:72\n",
      "Accuracy: 90.5602 | Precision: 44.1176 | Recall: 17.2414 | F1: 24.7934 | Specificity: 97.8335 | MCC: 0.6171\n",
      "\n",
      "TP:12,TN:905,FP:22,FN:25\n",
      "Accuracy: 95.1245 | Precision: 35.2941 | Recall: 32.4324 | F1: 33.8028 | Specificity: 97.6268 | MCC: 0.6565\n",
      "\n",
      "TP:3,TN:927,FP:31,FN:3\n",
      "Accuracy: 96.473 | Precision: 8.8235 | Recall: 50.0 | F1: 15.0 | Specificity: 96.7641 | MCC: 0.5997\n",
      "\n",
      "TP:8,TN:925,FP:26,FN:5\n",
      "Accuracy: 96.7842 | Precision: 23.5294 | Recall: 61.5385 | F1: 34.0426 | Specificity: 97.266 | MCC: 0.6838\n",
      "\n",
      "TP:26,TN:786,FP:8,FN:144\n",
      "Accuracy: 84.2324 | Precision: 76.4706 | Recall: 15.2941 | F1: 25.4902 | Specificity: 98.9924 | MCC: 0.6476\n",
      "\n",
      "TP:2,TN:927,FP:32,FN:3\n",
      "Accuracy: 96.3693 | Precision: 5.8824 | Recall: 40.0 | F1: 10.2564 | Specificity: 96.6632 | MCC: 0.5714\n",
      "\n",
      "TP:3,TN:916,FP:31,FN:14\n",
      "Accuracy: 95.332 | Precision: 8.8235 | Recall: 17.6471 | F1: 11.7647 | Specificity: 96.7265 | MCC: 0.5513\n",
      "\n",
      "TP:12,TN:879,FP:22,FN:51\n",
      "Accuracy: 92.4274 | Precision: 35.2941 | Recall: 19.0476 | F1: 24.7423 | Specificity: 97.5583 | MCC: 0.6112\n",
      "\n",
      "TP:11,TN:908,FP:23,FN:22\n",
      "Accuracy: 95.332 | Precision: 32.3529 | Recall: 33.3333 | F1: 32.8358 | Specificity: 97.5295 | MCC: 0.6521\n",
      "\n",
      "TP:10,TN:895,FP:20,FN:39\n",
      "Accuracy: 93.8797 | Precision: 33.3333 | Recall: 20.4082 | F1: 25.3165 | Specificity: 97.8142 | MCC: 0.6153\n",
      "\n",
      "TP:33,TN:661,FP:1,FN:269\n",
      "Accuracy: 71.9917 | Precision: 97.0588 | Recall: 10.9272 | F1: 19.6429 | Specificity: 99.8489 | MCC: 0.6355\n",
      "\n",
      "TP:6,TN:927,FP:25,FN:6\n",
      "Accuracy: 96.7842 | Precision: 19.3548 | Recall: 50.0 | F1: 27.907 | Specificity: 97.3739 | MCC: 0.6489\n",
      "\n",
      "TP:7,TN:914,FP:27,FN:16\n",
      "Accuracy: 95.5394 | Precision: 20.5882 | Recall: 30.4348 | F1: 24.5614 | Specificity: 97.1307 | MCC: 0.614\n",
      "\n",
      "TP:1,TN:934,FP:29,FN:0\n",
      "Accuracy: 96.9917 | Precision: 3.3333 | Recall: 100.0 | F1: 6.4516 | Specificity: 96.9886 | MCC: 0.5899\n",
      "\n",
      "TP:8,TN:910,FP:22,FN:24\n",
      "Accuracy: 95.2282 | Precision: 26.6667 | Recall: 25.0 | F1: 25.8065 | Specificity: 97.6395 | MCC: 0.6168\n",
      "\n",
      "TP:9,TN:921,FP:25,FN:9\n",
      "Accuracy: 96.473 | Precision: 26.4706 | Recall: 50.0 | F1: 34.6154 | Specificity: 97.3573 | MCC: 0.6738\n",
      "\n",
      "TP:8,TN:913,FP:26,FN:17\n",
      "Accuracy: 95.5394 | Precision: 23.5294 | Recall: 32.0 | F1: 27.1186 | Specificity: 97.2311 | MCC: 0.6259\n",
      "\n",
      "TP:27,TN:840,FP:7,FN:90\n",
      "Accuracy: 89.9378 | Precision: 79.4118 | Recall: 23.0769 | F1: 35.7616 | Specificity: 99.1736 | MCC: 0.697\n",
      "\n",
      "TP:24,TN:830,FP:10,FN:100\n",
      "Accuracy: 88.5892 | Precision: 70.5882 | Recall: 19.3548 | F1: 30.3797 | Specificity: 98.8095 | MCC: 0.6648\n",
      "\n",
      "TP:9,TN:922,FP:21,FN:12\n",
      "Accuracy: 96.5768 | Precision: 30.0 | Recall: 42.8571 | F1: 35.2941 | Specificity: 97.7731 | MCC: 0.6708\n",
      "\n",
      "TP:27,TN:802,FP:7,FN:128\n",
      "Accuracy: 85.9959 | Precision: 79.4118 | Recall: 17.4194 | F1: 28.5714 | Specificity: 99.1347 | MCC: 0.6648\n",
      "\n",
      "TP:14,TN:885,FP:20,FN:45\n",
      "Accuracy: 93.2573 | Precision: 41.1765 | Recall: 23.7288 | F1: 30.1075 | Specificity: 97.7901 | MCC: 0.6398\n",
      "\n",
      "TP:9,TN:916,FP:25,FN:14\n",
      "Accuracy: 95.9544 | Precision: 26.4706 | Recall: 39.1304 | F1: 31.5789 | Specificity: 97.3433 | MCC: 0.6509\n",
      "\n",
      "TP:10,TN:905,FP:24,FN:25\n",
      "Accuracy: 94.917 | Precision: 29.4118 | Recall: 28.5714 | F1: 28.9855 | Specificity: 97.4166 | MCC: 0.6318\n",
      "\n",
      "TP:11,TN:900,FP:23,FN:30\n",
      "Accuracy: 94.5021 | Precision: 32.3529 | Recall: 26.8293 | F1: 29.3333 | Specificity: 97.5081 | MCC: 0.6331\n",
      "\n",
      "TP:27,TN:830,FP:7,FN:100\n",
      "Accuracy: 88.9004 | Precision: 79.4118 | Recall: 21.2598 | F1: 33.5404 | Specificity: 99.1637 | MCC: 0.6872\n",
      "\n",
      "TP:29,TN:838,FP:5,FN:92\n",
      "Accuracy: 89.9378 | Precision: 85.2941 | Recall: 23.9669 | F1: 37.4194 | Specificity: 99.4069 | MCC: 0.7099\n",
      "\n",
      "TP:18,TN:863,FP:16,FN:67\n",
      "Accuracy: 91.39 | Precision: 52.9412 | Recall: 21.1765 | F1: 30.2521 | Specificity: 98.1797 | MCC: 0.6488\n",
      "\n",
      "TP:5,TN:914,FP:29,FN:16\n",
      "Accuracy: 95.332 | Precision: 14.7059 | Recall: 23.8095 | F1: 18.1818 | Specificity: 96.9247 | MCC: 0.582\n",
      "\n",
      "TP:14,TN:895,FP:20,FN:35\n",
      "Accuracy: 94.2946 | Precision: 41.1765 | Recall: 28.5714 | F1: 33.7349 | Specificity: 97.8142 | MCC: 0.6571\n",
      "\n",
      "TP:6,TN:921,FP:28,FN:9\n",
      "Accuracy: 96.1618 | Precision: 17.6471 | Recall: 40.0 | F1: 24.4898 | Specificity: 97.0495 | MCC: 0.6243\n",
      "\n",
      "TP:6,TN:925,FP:28,FN:5\n",
      "Accuracy: 96.5768 | Precision: 17.6471 | Recall: 54.5455 | F1: 26.6667 | Specificity: 97.0619 | MCC: 0.6486\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP:5,TN:927,FP:29,FN:3\n",
      "Accuracy: 96.6805 | Precision: 14.7059 | Recall: 62.5 | F1: 23.8095 | Specificity: 96.9665 | MCC: 0.6462\n",
      "\n",
      "TP:14,TN:905,FP:20,FN:25\n",
      "Accuracy: 95.332 | Precision: 41.1765 | Recall: 35.8974 | F1: 38.3562 | Specificity: 97.8378 | MCC: 0.6802\n",
      "\n",
      "TP:17,TN:903,FP:17,FN:27\n",
      "Accuracy: 95.4357 | Precision: 50.0 | Recall: 38.6364 | F1: 43.5897 | Specificity: 98.1522 | MCC: 0.7081\n",
      "\n",
      "TP:6,TN:926,FP:28,FN:4\n",
      "Accuracy: 96.6805 | Precision: 17.6471 | Recall: 60.0 | F1: 27.2727 | Specificity: 97.065 | MCC: 0.6567\n",
      "\n",
      "TP:19,TN:870,FP:15,FN:60\n",
      "Accuracy: 92.2199 | Precision: 55.8824 | Recall: 24.0506 | F1: 33.6283 | Specificity: 98.3051 | MCC: 0.6662\n",
      "\n",
      "TP:4,TN:922,FP:30,FN:8\n",
      "Accuracy: 96.0581 | Precision: 11.7647 | Recall: 33.3333 | F1: 17.3913 | Specificity: 96.8487 | MCC: 0.5907\n",
      "\n",
      "TP:8,TN:914,FP:26,FN:16\n",
      "Accuracy: 95.6432 | Precision: 23.5294 | Recall: 33.3333 | F1: 27.5862 | Specificity: 97.234 | MCC: 0.6291\n",
      "\n",
      "TP:1,TN:923,FP:29,FN:11\n",
      "Accuracy: 95.8506 | Precision: 3.3333 | Recall: 8.3333 | F1: 4.7619 | Specificity: 96.9538 | MCC: 0.5169\n",
      "\n",
      "TP:5,TN:923,FP:29,FN:7\n",
      "Accuracy: 96.2656 | Precision: 14.7059 | Recall: 41.6667 | F1: 21.7391 | Specificity: 96.9538 | MCC: 0.6161\n",
      "\n",
      "TP:1,TN:925,FP:33,FN:5\n",
      "Accuracy: 96.0581 | Precision: 2.9412 | Recall: 16.6667 | F1: 5.0 | Specificity: 96.5553 | MCC: 0.5282\n",
      "\n",
      "TP:11,TN:901,FP:23,FN:29\n",
      "Accuracy: 94.6058 | Precision: 32.3529 | Recall: 27.5 | F1: 29.7297 | Specificity: 97.5108 | MCC: 0.6352\n",
      "\n",
      "TP:3,TN:895,FP:27,FN:39\n",
      "Accuracy: 93.1535 | Precision: 10.0 | Recall: 7.1429 | F1: 8.3333 | Specificity: 97.0716 | MCC: 0.5248\n",
      "\n",
      "TP:20,TN:886,FP:14,FN:44\n",
      "Accuracy: 93.9834 | Precision: 58.8235 | Recall: 31.25 | F1: 40.8163 | Specificity: 98.4444 | MCC: 0.7004\n",
      "\n",
      "TP:11,TN:909,FP:23,FN:21\n",
      "Accuracy: 95.4357 | Precision: 32.3529 | Recall: 34.375 | F1: 33.3333 | Specificity: 97.5322 | MCC: 0.6549\n",
      "\n",
      "TP:5,TN:927,FP:29,FN:3\n",
      "Accuracy: 96.6805 | Precision: 14.7059 | Recall: 62.5 | F1: 23.8095 | Specificity: 96.9665 | MCC: 0.6462\n",
      "\n",
      "TP:17,TN:843,FP:17,FN:87\n",
      "Accuracy: 89.2116 | Precision: 50.0 | Recall: 16.3462 | F1: 24.6377 | Specificity: 98.0233 | MCC: 0.6208\n",
      "\n",
      "TP:5,TN:923,FP:29,FN:7\n",
      "Accuracy: 96.2656 | Precision: 14.7059 | Recall: 41.6667 | F1: 21.7391 | Specificity: 96.9538 | MCC: 0.6161\n",
      "\n",
      "TP:4,TN:904,FP:27,FN:29\n",
      "Accuracy: 94.1909 | Precision: 12.9032 | Recall: 12.1212 | F1: 12.5 | Specificity: 97.0999 | MCC: 0.5475\n",
      "\n",
      "TP:19,TN:869,FP:15,FN:61\n",
      "Accuracy: 92.1162 | Precision: 55.8824 | Recall: 23.75 | F1: 33.3333 | Specificity: 98.3032 | MCC: 0.6649\n",
      "\n",
      "TP:8,TN:922,FP:26,FN:8\n",
      "Accuracy: 96.473 | Precision: 23.5294 | Recall: 50.0 | F1: 32.0 | Specificity: 97.2574 | MCC: 0.6637\n",
      "\n",
      "TP:19,TN:867,FP:15,FN:63\n",
      "Accuracy: 91.9087 | Precision: 55.8824 | Recall: 23.1707 | F1: 32.7586 | Specificity: 98.2993 | MCC: 0.6624\n",
      "\n",
      "TP:31,TN:695,FP:3,FN:235\n",
      "Accuracy: 75.3112 | Precision: 91.1765 | Recall: 11.6541 | F1: 20.6667 | Specificity: 99.5702 | MCC: 0.636\n",
      "\n",
      "TP:34,TN:524,FP:0,FN:406\n",
      "Accuracy: 57.8838 | Precision: 100.0 | Recall: 7.7273 | F1: 14.346 | Specificity: 100.0 | MCC: 0.6043\n",
      "\n",
      "TP:16,TN:906,FP:18,FN:24\n",
      "Accuracy: 95.6432 | Precision: 47.0588 | Recall: 40.0 | F1: 43.2432 | Specificity: 98.0519 | MCC: 0.7057\n",
      "\n",
      "TP:10,TN:916,FP:24,FN:14\n",
      "Accuracy: 96.0581 | Precision: 29.4118 | Recall: 41.6667 | F1: 34.4828 | Specificity: 97.4468 | MCC: 0.6652\n",
      "\n",
      "TP:9,TN:913,FP:25,FN:17\n",
      "Accuracy: 95.6432 | Precision: 26.4706 | Recall: 34.6154 | F1: 30.0 | Specificity: 97.3348 | MCC: 0.6403\n",
      "\n",
      "TP:23,TN:828,FP:11,FN:102\n",
      "Accuracy: 88.278 | Precision: 67.6471 | Recall: 18.4 | F1: 28.9308 | Specificity: 98.6889 | MCC: 0.6556\n",
      "\n",
      "TP:15,TN:890,FP:19,FN:40\n",
      "Accuracy: 93.8797 | Precision: 44.1176 | Recall: 27.2727 | F1: 33.7079 | Specificity: 97.9098 | MCC: 0.6583\n",
      "\n",
      "TP:14,TN:897,FP:20,FN:33\n",
      "Accuracy: 94.5021 | Precision: 41.1765 | Recall: 29.7872 | F1: 34.5679 | Specificity: 97.819 | MCC: 0.6611\n",
      "\n",
      "TP:4,TN:922,FP:30,FN:8\n",
      "Accuracy: 96.0581 | Precision: 11.7647 | Recall: 33.3333 | F1: 17.3913 | Specificity: 96.8487 | MCC: 0.5907\n",
      "\n",
      "TP:2,TN:929,FP:32,FN:1\n",
      "Accuracy: 96.5768 | Precision: 5.8824 | Recall: 66.6667 | F1: 10.8108 | Specificity: 96.6701 | MCC: 0.5956\n",
      "\n",
      "TP:26,TN:796,FP:8,FN:134\n",
      "Accuracy: 85.2697 | Precision: 76.4706 | Recall: 16.25 | F1: 26.8041 | Specificity: 99.005 | MCC: 0.6538\n",
      "\n",
      "TP:9,TN:898,FP:25,FN:32\n",
      "Accuracy: 94.0871 | Precision: 26.4706 | Recall: 21.9512 | F1: 24.0 | Specificity: 97.2914 | MCC: 0.6053\n",
      "\n",
      "TP:19,TN:829,FP:15,FN:101\n",
      "Accuracy: 87.9668 | Precision: 55.8824 | Recall: 15.8333 | F1: 24.6753 | Specificity: 98.2227 | MCC: 0.6258\n",
      "\n",
      "TP:16,TN:887,FP:18,FN:43\n",
      "Accuracy: 93.6722 | Precision: 47.0588 | Recall: 27.1186 | F1: 34.4086 | Specificity: 98.011 | MCC: 0.6633\n",
      "\n",
      "TP:2,TN:925,FP:28,FN:9\n",
      "Accuracy: 96.1618 | Precision: 6.6667 | Recall: 18.1818 | F1: 9.7561 | Specificity: 97.0619 | MCC: 0.5466\n",
      "\n",
      "TP:7,TN:922,FP:27,FN:8\n",
      "Accuracy: 96.3693 | Precision: 20.5882 | Recall: 46.6667 | F1: 28.5714 | Specificity: 97.1549 | MCC: 0.647\n",
      "\n",
      "TP:3,TN:923,FP:31,FN:7\n",
      "Accuracy: 96.0581 | Precision: 8.8235 | Recall: 30.0 | F1: 13.6364 | Specificity: 96.7505 | MCC: 0.5735\n",
      "\n",
      "TP:4,TN:914,FP:30,FN:16\n",
      "Accuracy: 95.2282 | Precision: 11.7647 | Recall: 20.0 | F1: 14.8148 | Specificity: 96.822 | MCC: 0.565\n",
      "\n",
      "TP:8,TN:917,FP:26,FN:13\n",
      "Accuracy: 95.9544 | Precision: 23.5294 | Recall: 38.0952 | F1: 29.0909 | Specificity: 97.2428 | MCC: 0.6398\n",
      "\n",
      "TP:5,TN:919,FP:29,FN:11\n",
      "Accuracy: 95.8506 | Precision: 14.7059 | Recall: 31.25 | F1: 20.0 | Specificity: 96.9409 | MCC: 0.5976\n",
      "\n",
      "TP:15,TN:901,FP:19,FN:29\n",
      "Accuracy: 95.0207 | Precision: 44.1176 | Recall: 34.0909 | F1: 38.4615 | Specificity: 97.9348 | MCC: 0.6812\n",
      "\n",
      "TP:19,TN:858,FP:15,FN:72\n",
      "Accuracy: 90.9751 | Precision: 55.8824 | Recall: 20.8791 | F1: 30.4 | Specificity: 98.2818 | MCC: 0.6519\n",
      "\n",
      "TP:5,TN:920,FP:26,FN:13\n",
      "Accuracy: 95.9544 | Precision: 16.129 | Recall: 27.7778 | F1: 20.4082 | Specificity: 97.2516 | MCC: 0.596\n",
      "\n",
      "TP:31,TN:524,FP:3,FN:406\n",
      "Accuracy: 57.5726 | Precision: 91.1765 | Recall: 7.0938 | F1: 13.1635 | Specificity: 99.4307 | MCC: 0.588\n",
      "\n",
      "TP:3,TN:922,FP:31,FN:8\n",
      "Accuracy: 95.9544 | Precision: 8.8235 | Recall: 27.2727 | F1: 13.3333 | Specificity: 96.7471 | MCC: 0.5692\n",
      "\n",
      "TP:34,TN:627,FP:0,FN:303\n",
      "Accuracy: 68.5685 | Precision: 100.0 | Recall: 10.089 | F1: 18.3288 | Specificity: 100.0 | MCC: 0.6304\n",
      "\n",
      "TP:34,TN:171,FP:0,FN:759\n",
      "Accuracy: 21.2656 | Precision: 100.0 | Recall: 4.2875 | F1: 8.2225 | Specificity: 100.0 | MCC: 0.5444\n",
      "\n",
      "TP:2,TN:922,FP:32,FN:8\n",
      "Accuracy: 95.8506 | Precision: 5.8824 | Recall: 20.0 | F1: 9.0909 | Specificity: 96.6457 | MCC: 0.5457\n",
      "\n",
      "TP:25,TN:846,FP:9,FN:84\n",
      "Accuracy: 90.3527 | Precision: 73.5294 | Recall: 22.9358 | F1: 34.965 | Specificity: 98.9474 | MCC: 0.6878\n",
      "\n",
      "TP:22,TN:820,FP:12,FN:110\n",
      "Accuracy: 87.3444 | Precision: 64.7059 | Recall: 16.6667 | F1: 26.506 | Specificity: 98.5577 | MCC: 0.6419\n",
      "\n",
      "TP:22,TN:818,FP:12,FN:112\n",
      "Accuracy: 87.1369 | Precision: 64.7059 | Recall: 16.4179 | F1: 26.1905 | Specificity: 98.5542 | MCC: 0.6404\n",
      "\n",
      "TP:14,TN:905,FP:20,FN:25\n",
      "Accuracy: 95.332 | Precision: 41.1765 | Recall: 35.8974 | F1: 38.3562 | Specificity: 97.8378 | MCC: 0.6802\n",
      "\n",
      "TP:3,TN:896,FP:27,FN:38\n",
      "Accuracy: 93.2573 | Precision: 10.0 | Recall: 7.3171 | F1: 8.4507 | Specificity: 97.0748 | MCC: 0.5255\n",
      "\n",
      "TP:29,TN:746,FP:5,FN:184\n",
      "Accuracy: 80.3942 | Precision: 85.2941 | Recall: 13.615 | F1: 23.4818 | Specificity: 99.3342 | MCC: 0.6456\n",
      "\n",
      "TP:10,TN:906,FP:24,FN:24\n",
      "Accuracy: 95.0207 | Precision: 29.4118 | Recall: 29.4118 | F1: 29.4118 | Specificity: 97.4194 | MCC: 0.6342\n",
      "\n",
      "TP:20,TN:900,FP:14,FN:30\n",
      "Accuracy: 95.4357 | Precision: 58.8235 | Recall: 40.0 | F1: 47.619 | Specificity: 98.4683 | MCC: 0.7312\n",
      "\n",
      "TP:12,TN:896,FP:22,FN:34\n",
      "Accuracy: 94.1909 | Precision: 35.2941 | Recall: 26.087 | F1: 30.0 | Specificity: 97.6035 | MCC: 0.6369\n",
      "\n",
      "TP:4,TN:900,FP:27,FN:33\n",
      "Accuracy: 93.7759 | Precision: 12.9032 | Recall: 10.8108 | F1: 11.7647 | Specificity: 97.0874 | MCC: 0.543\n",
      "\n",
      "TP:2,TN:926,FP:32,FN:4\n",
      "Accuracy: 96.2656 | Precision: 5.8824 | Recall: 33.3333 | F1: 10.0 | Specificity: 96.6597 | MCC: 0.5639\n",
      "\n",
      "TP:21,TN:860,FP:13,FN:70\n",
      "Accuracy: 91.39 | Precision: 61.7647 | Recall: 23.0769 | F1: 33.6 | Specificity: 98.5109 | MCC: 0.6711\n",
      "\n",
      "TP:10,TN:922,FP:24,FN:8\n",
      "Accuracy: 96.6805 | Precision: 29.4118 | Recall: 55.5556 | F1: 38.4615 | Specificity: 97.463 | MCC: 0.6945\n",
      "\n",
      "TP:14,TN:889,FP:20,FN:41\n",
      "Accuracy: 93.6722 | Precision: 41.1765 | Recall: 25.4545 | F1: 31.4607 | Specificity: 97.7998 | MCC: 0.6462\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP:8,TN:899,FP:26,FN:31\n",
      "Accuracy: 94.0871 | Precision: 23.5294 | Recall: 20.5128 | F1: 21.9178 | Specificity: 97.1892 | MCC: 0.5945\n",
      "\n",
      "TP:5,TN:926,FP:29,FN:4\n",
      "Accuracy: 96.5768 | Precision: 14.7059 | Recall: 55.5556 | F1: 23.2558 | Specificity: 96.9634 | MCC: 0.6369\n",
      "\n",
      "TP:7,TN:911,FP:27,FN:19\n",
      "Accuracy: 95.2282 | Precision: 20.5882 | Recall: 26.9231 | F1: 23.3333 | Specificity: 97.1215 | MCC: 0.6056\n",
      "\n",
      "TP:27,TN:823,FP:7,FN:107\n",
      "Accuracy: 88.1743 | Precision: 79.4118 | Recall: 20.1493 | F1: 32.1429 | Specificity: 99.1566 | MCC: 0.681\n",
      "\n",
      "TP:3,TN:924,FP:31,FN:6\n",
      "Accuracy: 96.1618 | Precision: 8.8235 | Recall: 33.3333 | F1: 13.9535 | Specificity: 96.7539 | MCC: 0.5784\n",
      "\n",
      "TP:5,TN:910,FP:29,FN:20\n",
      "Accuracy: 94.917 | Precision: 14.7059 | Recall: 20.0 | F1: 16.9492 | Specificity: 96.9116 | MCC: 0.5729\n",
      "\n",
      "TP:16,TN:893,FP:14,FN:41\n",
      "Accuracy: 94.2946 | Precision: 53.3333 | Recall: 28.0702 | F1: 36.7816 | Specificity: 98.4564 | MCC: 0.6802\n",
      "\n",
      "TP:24,TN:899,FP:6,FN:35\n",
      "Accuracy: 95.7469 | Precision: 80.0 | Recall: 40.678 | F1: 53.9326 | Specificity: 99.337 | MCC: 0.7762\n",
      "\n",
      "TP:8,TN:909,FP:26,FN:21\n",
      "Accuracy: 95.1245 | Precision: 23.5294 | Recall: 27.5862 | F1: 25.3968 | Specificity: 97.2193 | MCC: 0.6149\n",
      "\n",
      "TP:15,TN:894,FP:15,FN:40\n",
      "Accuracy: 94.2946 | Precision: 50.0 | Recall: 27.2727 | F1: 35.2941 | Specificity: 98.3498 | MCC: 0.6711\n",
      "\n",
      "TP:28,TN:754,FP:6,FN:176\n",
      "Accuracy: 81.1203 | Precision: 82.3529 | Recall: 13.7255 | F1: 23.5294 | Specificity: 99.2105 | MCC: 0.6432\n",
      "\n",
      "TP:27,TN:761,FP:7,FN:169\n",
      "Accuracy: 81.7427 | Precision: 79.4118 | Recall: 13.7755 | F1: 23.4783 | Specificity: 99.0885 | MCC: 0.6403\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Evaluation\n",
    "\n",
    "# Specify a path\n",
    "PATH = \"../results/entire_model_V2.pt\"\n",
    "conf = '../results/state_dict_model_V2.pt'\n",
    "\n",
    "# Load\n",
    "model = GCN(964,200,964)\n",
    "model.load_state_dict(torch.load(conf))\n",
    "model.eval()\n",
    "print('\\nEvaluating \\n')\n",
    "\n",
    "#f = open(\"Eval.txt\", \"a\")\n",
    "\n",
    "def predict(g1, g2):  # graph1, graph2\n",
    "    pred1 = model(g1, g1.ndata['PSE'].float())\n",
    "    pred2 = model(g2, g2.ndata['PSE'].float())\n",
    "    pred = F.normalize(pred1+pred2)/2\n",
    "    return(pred)\n",
    "\n",
    "all_acc = []\n",
    "all_prec = []\n",
    "all_spec = []\n",
    "all_mcc = []\n",
    "for batched_graph, labels in eval_dataloader:\n",
    "    g1 = batched_graph[0]\n",
    "    g2 = batched_graph[1]\n",
    "    pred = predict(g1, g2)\n",
    "    #print(pred)\n",
    "    #print(labels)\n",
    "\n",
    "    # Threshold \n",
    "    tr = 0.5*pred.mean().tolist()\n",
    "    TP = 0\n",
    "    FP = 0\n",
    "    TN = 0\n",
    "    FN = 0\n",
    "    for i in range(len(labels)):\n",
    "        for j in range(len(labels[i])):\n",
    "            if labels[i][j] == 1 and pred[i][j] >= tr:\n",
    "                TP += 1\n",
    "            elif labels[i][j] == 1 and pred[i][j] < tr:\n",
    "                FN += 1\n",
    "            elif labels[i][j] == 0 and pred[i][j] >= tr:\n",
    "                FP += 1\n",
    "            elif labels[i][j] == 0 and pred[i][j] < tr:\n",
    "                TN += 1\n",
    "            else:\n",
    "                pass\n",
    "            \n",
    "    # Validation metrics        \n",
    "    acc = ((TP+TN)*100)/(TP+FP+FN+TN)\n",
    "    prec = (TP*100)/(TP+FP)\n",
    "    recall = (TP*100)/(TP+FN)\n",
    "    F1 = 2*(recall*prec)/(recall+prec)\n",
    "    spec = (TN*100)/(FP+TN)\n",
    "    mcc = (TP*TN-FP*FN)/((TP+FN)*(TP+FP)*(TN+FP)*(TN+FN))**0.5\n",
    "    mcc2 = (mcc+1)/2\n",
    "    #sim = ((F.cosine_similarity(pred.float(),labels.float())).mean().tolist())*100\n",
    "    \n",
    "    x = [TP,TN,FP,FN]\n",
    "    print('TP:%s,TN:%s,FP:%s,FN:%s' % (x[0],x[1],x[2],x[3]))\n",
    "    msg2 = 'Accuracy: %s | Precision: %s | Recall: %s | F1: %s | Specificity: %s | MCC: %s\\n' %(\n",
    "            round(acc,4),round(prec,4),round(recall,4),round(F1,4),round(spec,4),round(mcc2,4))\n",
    "    all_acc.append(acc)\n",
    "    all_prec.append(prec)\n",
    "    all_spec.append(spec)\n",
    "    all_mcc.append(mcc2)\n",
    "    print(msg2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "91.48335036176738"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(all_acc)/len(all_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39.9514333209374"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(all_prec)/len(all_prec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99.37736665920714"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(all_spec)/len(all_spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6244535919931011"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(all_mcc)/len(all_mcc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy import interp\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute ROC curve and ROC area for each class\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "\n",
    "# Compute micro-average ROC curve and ROC area\n",
    "fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel())\n",
    "roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr[2], tpr[2], color='darkorange',\n",
    "         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2])\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic example')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}