Merge pull request ML4ITS#12 from ML4ITS/experiments

Experiments

.gitignore

@@ -2,15 +2,12 @@
 __pycache__/
 .idea/
 .DS_Store
-datasets/ecg_data.txt
+datasets/data/
 datasets/ServerMachineDataset/processed/
-/datasets/data/processed/
-/datasets/data/test/
-/datasets/data/train/
-eda/.ipynb_checkpoints/
-plots/
-*.pkl
+visualization/.ipynb_checkpoints/
 models/
-output/smd/*/logs
-output/smd/*/preds.pkl
-output/smd/*/anomaly_preds.pkl
+**.pkl
+**.npy
+**/logs
+*.pkl
+*.npy

args.py

@@ -1,20 +1,26 @@
 import argparse
 
 
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("Boolean value expected.")
+
+
 def get_parser():
     parser = argparse.ArgumentParser()
 
     # Data params
     parser.add_argument("--dataset", type=str.upper, default="SMD")
-    parser.add_argument(
-        "--group",
-        type=str,
-        default="1-1",
-        help="Required for smd dataset. <group_index>-<index>",
-    )
+    parser.add_argument("--group", type=str, default="1-1", help="Required for SMD dataset. <group_index>-<index>")
     parser.add_argument("--lookback", type=int, default=100)
     parser.add_argument("--horizon", type=int, default=1)
-    parser.add_argument("--target_col", type=int, default=None)
+    parser.add_argument("--do_preprocess", type=str2bool, default=False)
 
     # Model params
     parser.add_argument("--kernel_size", type=int, default=7)
@@ -31,27 +37,19 @@ def get_parser():
     parser.add_argument("--bs", type=int, default=256)
     parser.add_argument("--init_lr", type=float, default=1e-3)
     parser.add_argument("--val_split", type=float, default=0.1)
-    parser.add_argument("--shuffle_dataset", type=bool, default=True)
+    parser.add_argument("--shuffle_dataset", type=str2bool, default=True)
     parser.add_argument("--dropout", type=float, default=0.3)
-    parser.add_argument("--use_cuda", type=bool, default=True)
+    parser.add_argument("--use_cuda", type=str2bool, default=True)
     parser.add_argument("--model_path", type=str, default="models")
     parser.add_argument("--print_every", type=int, default=1)
 
     # Predictor args
-    parser.add_argument(
-        "--save_scores",
-        type=bool,
-        default=True,
-        help="To save anomaly scores predicted.",
-    )
-    parser.add_argument(
-        "--load_scores",
-        type=bool,
-        default=False,
-        help="To use already computed anomaly scores",
-    )
+    parser.add_argument("--save_scores", type=str2bool, default=True, help="To save anomaly scores predicted.")
+    parser.add_argument("--load_scores", type=str2bool, default=False, help="To use already computed anomaly scores")
     parser.add_argument("--gamma", type=float, default=1)
     parser.add_argument("--level", type=float, default=None)
+    parser.add_argument("--q", type=float, default=1e-3)
+    parser.add_argument("--use_mov_av", type=str2bool, default=False)
 
     # Other
     parser.add_argument("--comment", type=str, default="")

bash_scripts/copy_files_from_ssh.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+port=11726
+[email protected]
+
+scp -P $port -r $ssh:~/MTAD-GAT/models ./
+
+for i in {1..8}
+do
+    scp -P $port -r $ssh:~/MTAD-GAT/output/SMD/1-$i ./output/SMD/
+done
+
+for i in {1..9}
+do
+    scp -P $port -r $ssh:~/MTAD-GAT/output/SMD/2-$i ./output/SMD/
+done
+
+for i in {1..11}
+do
+    scp -P $port -r $ssh:~/MTAD-GAT/output/SMD/3-$i ./output/SMD/
+done

bash_scripts/predict_smd.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+python predict.py --dataset SMD --group 1-1 --model 17032021_010924 --load_scores True
+python predict.py --dataset SMD --group 1-2 --model 17032021_012546 --load_scores True
+python predict.py --dataset SMD --group 1-3 --model 17032021_013914 --load_scores True
+python predict.py --dataset SMD --group 1-4 --model 17032021_015244 --load_scores True
+python predict.py --dataset SMD --group 1-5 --model 17032021_020613 --load_scores True
+python predict.py --dataset SMD --group 1-6 --model 17032021_021942 --load_scores True
+python predict.py --dataset SMD --group 1-7 --model 17032021_023323 --load_scores True
+python predict.py --dataset SMD --group 1-8 --model 17032021_024657 --load_scores True
+python predict.py --dataset SMD --group 2-1 --model 17032021_030025 --load_scores True
+python predict.py --dataset SMD --group 2-2 --model 17032021_031410 --load_scores True
+python predict.py --dataset SMD --group 2-3 --model 17032021_032905 --load_scores True
+python predict.py --dataset SMD --group 2-4 --model 17032021_034448 --load_scores True
+python predict.py --dataset SMD --group 2-5 --model 17032021_035857 --load_scores True
+python predict.py --dataset SMD --group 2-6 --model 17032021_041234 --load_scores True
+python predict.py --dataset SMD --group 2-7 --model 17032021_042906 --load_scores True
+python predict.py --dataset SMD --group 2-8 --model 17032021_044240 --load_scores True
+python predict.py --dataset SMD --group 2-9 --model 17032021_045614 --load_scores True
+python predict.py --dataset SMD --group 3-1 --model 17032021_051238 --load_scores True
+python predict.py --dataset SMD --group 3-2 --model 17032021_052901 --load_scores True
+python predict.py --dataset SMD --group 3-3 --model 17032021_054227 --load_scores True
+python predict.py --dataset SMD --group 3-4 --model 17032021_055608 --load_scores True
+python predict.py --dataset SMD --group 3-5 --model 17032021_060937 --load_scores True
+python predict.py --dataset SMD --group 3-6 --model 17032021_062314 --load_scores True
+python predict.py --dataset SMD --group 3-7 --model 17032021_063934 --load_scores True
+python predict.py --dataset SMD --group 3-8 --model 17032021_065554 --load_scores True
+python predict.py --dataset SMD --group 3-9 --model 17032021_071215 --load_scores True
+python predict.py --dataset SMD --group 3-10 --model 17032021_072910 --load_scores True
+python predict.py --dataset SMD --group 3-11 --model 17032021_074315 --load_scores True

eval_methods.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from spot import SPOT
+from spot import SPOT, dSPOT
 
 
 def adjust_predicts(score, label, threshold, advance=1, pred=None, calc_latency=False):
@@ -87,28 +87,33 @@ def pot_eval(init_score, score, label, q=1e-3, level=0.99):
     s = SPOT(q)  # SPOT object
     s.fit(init_score, score)  # data import
     s.initialize(level=level, min_extrema=False)  # initialization step
-    # ret = s.run(dynamic=True)  # run
     ret = s.run(dynamic=False, with_alarm=False)  # much faster
+
+    # s = dSPOT(q, depth=300)  # SPOT object
+    # s.fit(init_score, score)  # data import
+    # s.initialize()  # initialization step
+    # ret = s.run()  # much faster
+
     print(len(ret["alarms"]))
     print(len(ret["thresholds"]))
 
     pot_th = np.mean(ret["thresholds"])
     pred, p_latency = adjust_predicts(score, label, pot_th, calc_latency=True)
     # pred = adjust_predicts(score, label, pot_th, advance=1, delay=30)
     p_t = calc_point2point(pred, label)
-    print("POT result: ", p_t, pot_th)
+    # print("POT result: ", p_t, pot_th)
     return {
-        "pot-f1": p_t[0],
-        "pot-precision": p_t[1],
-        "pot-recall": p_t[2],
-        "pot-TP": p_t[3],
-        "pot-TN": p_t[4],
-        "pot-FP": p_t[5],
-        "pot-FN": p_t[6],
-        "pot-threshold": pot_th,
-        "pot-latency": p_latency,
+        "f1": p_t[0],
+        "precision": p_t[1],
+        "recall": p_t[2],
+        "TP": p_t[3],
+        "TN": p_t[4],
+        "FP": p_t[5],
+        "FN": p_t[6],
+        "threshold": pot_th,
+        "latency": p_latency,
         "pred": pred,
-        "pot_thresholds": ret["thresholds"],
+        "thresholds": ret["thresholds"],
     }
 
 
@@ -119,6 +124,7 @@ def bf_search(score, label, start, end=None, step_num=1, display_freq=1, verbose
             list: list for results
             float: the `threshold` for best-f1
     """
+    print(f"Finding best f1-score by searching for threshold..")
     if step_num is None or end is None:
         end = start
         step_num = 1
@@ -128,21 +134,33 @@ def bf_search(score, label, start, end=None, step_num=1, display_freq=1, verbose
     threshold = search_lower_bound
     m = (-1.0, -1.0, -1.0)
     m_t = 0.0
+    m_l = 0
     for i in range(search_step):
         threshold += search_range / float(search_step)
-        target = calc_seq(score, label, threshold)
+        target, latency = calc_seq(score, label, threshold)
         if target[0] > m[0]:
             m_t = threshold
             m = target
+            m_l = latency
         if verbose and i % display_freq == 0:
             print("cur thr: ", threshold, target, m, m_t)
-    print(m, m_t)
-    return m, m_t
+
+    return {
+        "f1": m[0],
+        "precision": m[1],
+        "recall": m[2],
+        "TP": m[3],
+        "TN": m[4],
+        "FP": m[5],
+        "FN": m[6],
+        "threshold": m_t,
+        "latency": m_l,
+    }
 
 
 def calc_seq(score, label, threshold):
     """
     Calculate f1 score for a score sequence
     """
-    predict = adjust_predicts(score, label, threshold)
-    return calc_point2point(predict, label)
+    predict, latency = adjust_predicts(score, label, threshold, calc_latency=True)
+    return calc_point2point(predict, label), latency

mtad_gat.py

@@ -51,7 +51,7 @@ def __init__(
             gru_hid_dim,
             autoenc_n_layers,
             autoenc_hid_dim,
-            n_features,
+            out_dim,
             dropout,
             device,
         )

output/MSL/summary.txt

@@ -0,0 +1,35 @@
+{
+  "pred_args": {
+    "model_name": "23032021_213948",
+    "target_dims": [
+      0
+    ],
+    "level": 0.99,
+    "q": 0.001,
+    "use_mov_av": false,
+    "gamma": 1,
+    "save_path": "output/MSL"
+  },
+  "pot_result": {
+    "f1": 0.9060187838333649,
+    "precision": 0.9095509980410706,
+    "recall": 0.9025238206251304,
+    "TP": 7009.0,
+    "TN": 65166.0,
+    "FP": 697.0,
+    "FN": 757.0,
+    "threshold": 1.359734486050769,
+    "latency": 35.33321555594815
+  },
+  "bf_result": {
+    "f1": 0.9210810187184575,
+    "precision": 0.9077269305979908,
+    "recall": 0.9348441914307956,
+    "TP": 7260.0,
+    "TN": 65125.0,
+    "FP": 738.0,
+    "FN": 506.0,
+    "threshold": 1.3179999999999998,
+    "latency": 40.74180405869659
+  }
+}

output/SMAP/summary.txt

@@ -0,0 +1,35 @@
+{
+  "pred_args": {
+    "model_name": "24032021_175825",
+    "target_dims": [
+      0
+    ],
+    "level": 0.93,
+    "q": 0.01,
+    "use_mov_av": true,
+    "gamma": 1,
+    "save_path": "output/SMAP"
+  },
+  "pot_result": {
+    "f1": 0.7484403976489045,
+    "precision": 0.7565896395067176,
+    "recall": 0.7404746232374443,
+    "TP": 40501.0,
+    "TN": 359791.0,
+    "FP": 13030.0,
+    "FN": 14195.0,
+    "threshold": 0.1901731103119371,
+    "latency": 47.69689742894329
+  },
+  "bf_result": {
+    "f1": 0.8273023361879549,
+    "precision": 0.7454646779084775,
+    "recall": 0.9293366972119831,
+    "TP": 50831.0,
+    "TN": 355465.0,
+    "FP": 17356.0,
+    "FN": 3865.0,
+    "threshold": 0.1597,
+    "latency": 128.79085255096635
+  }
+}

output/SMD/1-1/summary.txt

@@ -0,0 +1,33 @@
+{
+  "pred_args": {
+    "model_name": "17032021_010924",
+    "target_dims": null,
+    "level": 0.995,
+    "q": 0.0001,
+    "use_mov_av": false,
+    "gamma": 1,
+    "save_path": "output/smd/1-1"
+  },
+  "pot_result": {
+    "f1": 0.9028100576310844,
+    "precision": 0.8228466682258806,
+    "recall": 0.9999999962880475,
+    "TP": 2694.0,
+    "TN": 25105.0,
+    "FP": 580.0,
+    "FN": 0.0,
+    "threshold": 0.06431315668483498,
+    "latency": 3.374957813027337
+  },
+  "bf_result": {
+    "f1": 0.9946375237059875,
+    "precision": 0.9900698749905483,
+    "recall": 0.9992576057934015,
+    "TP": 2692.0,
+    "TN": 25658.0,
+    "FP": 27.0,
+    "FN": 2.0,
+    "threshold": 0.1597,
+    "latency": 17.714032656676334
+  }
+}