From 629bb988e5e2e522764c77e41c4ec394e68b48a5 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 17:58:57 +0300
Subject: [PATCH 01/19] Fix formatting and direction of metric

---
 .../parameter_randomization_check.py            | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/xaib/metrics/feature_importance/parameter_randomization_check.py b/xaib/metrics/feature_importance/parameter_randomization_check.py
index 957e1df..5440e13 100644
--- a/xaib/metrics/feature_importance/parameter_randomization_check.py
+++ b/xaib/metrics/feature_importance/parameter_randomization_check.py
@@ -11,18 +11,21 @@ class ParameterRandomizationCheck(Metric):
     to the underlying model - whether it is
     sensitive to the changes in model
     """
-    def __init__(self, ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any) -> None:
+
+    def __init__(
+        self, ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, **kwargs)
         self._noisy_model = noisy_model
-        self.name = 'parameter_randomization_check'
-        self.direction = 'up'
+        self.name = "parameter_randomization_check"
+        self.direction = "down"
 
     def compute(
         self,
         expl: Explainer,
         batch_size: int = 1,
         expl_kwargs: Union[Dict[Any, Any], None] = None,
-        expl_noisy_kwargs: Union[Dict[Any, Any], None] = None
+        expl_noisy_kwargs: Union[Dict[Any, Any], None] = None,
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
@@ -32,10 +35,12 @@ def compute(
         diffs_expl = []
 
         for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
-            item = batch['item']
+            item = batch["item"]
 
             explanation_batch = expl.predict(item, self._model, **expl_kwargs)
-            noisy_explanation_batch = expl.predict(item, self._noisy_model, **expl_noisy_kwargs)
+            noisy_explanation_batch = expl.predict(
+                item, self._noisy_model, **expl_noisy_kwargs
+            )
 
             explanation_batch = minmax_normalize(explanation_batch)
             noisy_explanation_batch = minmax_normalize(noisy_explanation_batch)

From 502facf62293ec84762290ece45c4098d966f261 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 18:03:23 +0300
Subject: [PATCH 02/19] Autoformat utils

---
 xaib/evaluation/utils.py | 72 +++++++++++++++++++++++-----------------
 1 file changed, 41 insertions(+), 31 deletions(-)

diff --git a/xaib/evaluation/utils.py b/xaib/evaluation/utils.py
index 152f4e0..935f55b 100644
--- a/xaib/evaluation/utils.py
+++ b/xaib/evaluation/utils.py
@@ -1,12 +1,11 @@
-from typing import Dict, Any
 import os
+from typing import Any
 
 import numpy as np
+import pandas as pd
 from cascade import data as cdd
 from cascade import models as cdm
-
 from cascade.meta import MetaViewer
-import pandas as pd
 from plotly import graph_objects as go
 
 
@@ -21,6 +20,7 @@ def save(self, filepath: str, *args: Any, **kwargs: Any) -> None:
     def load(self, filepath: str, *args: Any, **kwargs: Any) -> None:
         return self._model.load(filepath, *args, **kwargs)
 
+
 class WrapperDataset(cdd.Modifier):
     def __init__(self, ds, name: str, *args: Any, **kwargs: Any) -> None:
         super().__init__(dataset=ds, *args, **kwargs)
@@ -28,15 +28,16 @@ def __init__(self, ds, name: str, *args: Any, **kwargs: Any) -> None:
 
 
 class NoiseApplier(cdd.Modifier):
-    def __init__(self, dataset, multiplier: float = 1., *args, **kwargs) -> None:
+    def __init__(self, dataset, multiplier: float = 1.0, *args, **kwargs) -> None:
         super().__init__(dataset, *args, **kwargs)
 
         self._multiplier = multiplier
 
     def __getitem__(self, index):
         item = self._dataset.__getitem__(index)
-        item['item'] = item['item'] \
-            + np.random.random(item['item'].shape) * self._multiplier
+        item["item"] = (
+            item["item"] + np.random.random(item["item"].shape) * self._multiplier
+        )
         return item
 
 
@@ -57,8 +58,9 @@ def __init__(self, dataset_length) -> None:
         self._dataset_length = dataset_length
 
     def kneighbors(self, x, n_neighbors):
-        return None, np.asarray([[np.random.randint(0, self._dataset_length)]
-            for _ in range(n_neighbors)])
+        return None, np.asarray(
+            [[np.random.randint(0, self._dataset_length)] for _ in range(n_neighbors)]
+        )
 
 
 class RandomNeighborsBaseline(cdm.BasicModel):
@@ -67,8 +69,6 @@ def __init__(self, dataset_length, *args, meta_prefix=None, **kwargs) -> None:
         self._pipeline = [KNeighborsTransformer(dataset_length)]
 
 
-
-
 def experiment(root, explainers, *args, batch_size=1, **kwargs):
     def wrapper(case_init):
         def wrap_case():
@@ -78,7 +78,9 @@ def wrap_case():
             line = repo.add_line()
 
             for name in explainers:
-                c.evaluate(name, explainers[name], *args, batch_size=batch_size, **kwargs)
+                c.evaluate(
+                    name, explainers[name], *args, batch_size=batch_size, **kwargs
+                )
                 line.save(c, only_meta=True)
 
         return wrap_case
@@ -87,37 +89,45 @@ def wrap_case():
 
 
 def visualize_results(path, output_path=None, title=None):
-    m = MetaViewer(path, filt={'type': 'model'})
+    m = MetaViewer(path, filt={"type": "model"})
 
     data = []
     for p in m:
-        for metric_name in p[0]['metrics']:
+        for metric_name in p[0]["metrics"]:
             data.append(
                 {
-                    'name': p[0]['params']['metric_params'][metric_name]['name'],
-                    'case': p[0]['params']['case'],
-                    'dataset': p[0]['params']['metric_params'][metric_name]['dataset'],
-                    'model': p[0]['params']['metric_params'][metric_name]['model'],
-                    'metric': metric_name,
-                    'direction': p[0]['params']['metric_params'][metric_name]['direction'],
-                    'value': p[0]['metrics'][metric_name]
+                    "name": p[0]["params"]["metric_params"][metric_name]["name"],
+                    "case": p[0]["params"]["case"],
+                    "dataset": p[0]["params"]["metric_params"][metric_name]["dataset"],
+                    "model": p[0]["params"]["metric_params"][metric_name]["model"],
+                    "metric": metric_name,
+                    "direction": p[0]["params"]["metric_params"][metric_name][
+                        "direction"
+                    ],
+                    "value": p[0]["metrics"][metric_name],
                 }
             )
 
     df = pd.DataFrame(data)
-    df = pd.pivot_table(df, values='value', columns=['name'], index=['dataset', 'model', 'case', 'metric', 'direction'])
+    df = pd.pivot_table(
+        df,
+        values="value",
+        columns=["name"],
+        index=["dataset", "model", "case", "metric", "direction"],
+    )
 
     df = df.reset_index()
-    df.loc[df['dataset'].duplicated(), 'dataset'] = ''
-    df.loc[df['model'].duplicated(), 'model'] = ''
-    df.loc[df['case'].duplicated(), 'case'] = ''
-    df.loc[df['metric'].duplicated(), 'metric'] = ''
-
-    fig = go.Figure(data=[go.Table(
-        header=dict(values=list(df.columns),
-                    align='left'),
-        cells=dict(values=[df[col] for col in df.columns],
-                align='left'))
+    df.loc[df["dataset"].duplicated(), "dataset"] = ""
+    df.loc[df["model"].duplicated(), "model"] = ""
+    df.loc[df["case"].duplicated(), "case"] = ""
+    df.loc[df["metric"].duplicated(), "metric"] = ""
+
+    fig = go.Figure(
+        data=[
+            go.Table(
+                header=dict(values=list(df.columns), align="left"),
+                cells=dict(values=[df[col] for col in df.columns], align="left"),
+            )
         ]
     )
     fig.update_layout(title=title)

From 1fd57d9eafe43204aaca8735951fc94986b7a7a9 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 19:00:20 +0300
Subject: [PATCH 03/19] Insert several simple plots

---
 xaib/docs/embed_plots.py | 28 ++++++------
 xaib/evaluation/utils.py | 93 ++++++++++++++++++++++++++++++++--------
 2 files changed, 88 insertions(+), 33 deletions(-)

diff --git a/xaib/docs/embed_plots.py b/xaib/docs/embed_plots.py
index 9d7762a..149b3ca 100644
--- a/xaib/docs/embed_plots.py
+++ b/xaib/docs/embed_plots.py
@@ -1,32 +1,32 @@
 import os
 import sys
 from bs4 import BeautifulSoup
+from plotly.io import to_html
 
 SCRIPT_DIR = os.path.dirname(__file__)
 BASE_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR))
 sys.path.append(BASE_DIR)
 from xaib.evaluation.utils import visualize_results
 
-RESULTS = ('feature_importance', 'example_selection')
+RESULTS = ("feature_importance", "example_selection")
 
-REPO_PATHS = [os.path.join(BASE_DIR, 'xaib', 'results', name)
-              for name in RESULTS]
-INDEX_PATH = os.path.join(BASE_DIR, 'xaib', 'docs', 'build', 'results.html')
+REPO_PATHS = [os.path.join(BASE_DIR, "xaib", "results", name) for name in RESULTS]
+INDEX_PATH = os.path.join(BASE_DIR, "xaib", "docs", "build", "results.html")
 
 for name, path in zip(RESULTS, REPO_PATHS):
-    title = 'Results of ' + ' '.join([a.capitalize() for a in name.split('_')]) + ' Methods'
+    figs = visualize_results(path)
 
-    fig = visualize_results(path, title=title)
-    htmlfig = fig.to_html(include_plotlyjs='cdn')
+    for name in figs:
+        htmlfig = to_html(figs[name], include_plotlyjs="cdn")
 
-    htmlfig = BeautifulSoup(htmlfig, "html.parser")
+        htmlfig = BeautifulSoup(htmlfig, "html.parser")
 
-    with open(INDEX_PATH, 'r', encoding='UTF-8') as f:
-        html = '\n'.join(f.readlines())
+        with open(INDEX_PATH, "r", encoding="UTF-8") as f:
+            html = "\n".join(f.readlines())
 
-    soup = BeautifulSoup(html, "html.parser")
+        soup = BeautifulSoup(html, "html.parser")
 
-    soup.findAll('footer', {'class': 'md-footer'})[0].insert_before(htmlfig)
+        soup.findAll("footer", {"class": "md-footer"})[0].insert_before(htmlfig)
 
-    with open(INDEX_PATH, 'w', encoding='UTF-8') as f:
-        f.write(soup.prettify())
+        with open(INDEX_PATH, "w", encoding="UTF-8") as f:
+            f.write(soup.prettify())
diff --git a/xaib/evaluation/utils.py b/xaib/evaluation/utils.py
index 935f55b..c23834f 100644
--- a/xaib/evaluation/utils.py
+++ b/xaib/evaluation/utils.py
@@ -6,6 +6,7 @@
 from cascade import data as cdd
 from cascade import models as cdm
 from cascade.meta import MetaViewer
+from plotly import express as px
 from plotly import graph_objects as go
 
 
@@ -88,7 +89,60 @@ def wrap_case():
     return wrapper
 
 
-def visualize_results(path, output_path=None, title=None):
+def table(df):
+    fig = go.Figure(
+        data=[
+            go.Table(
+                header=dict(values=list(df.columns), align="left"),
+                cells=dict(values=[df[col] for col in df.columns], align="left"),
+            )
+        ]
+    )
+    fig.update_layout(title="Table with metric values on all setups")
+
+    return fig
+
+
+def relative_bar(df):
+    metrics = df.metric.unique()
+
+    df["viz_value"] = df["value"]
+    for metric in metrics:
+        df.loc[df["metric"] == metric, "viz_value"] /= df.loc[
+            df["metric"] == metric, "value"
+        ].max()
+    df.loc[df["direction"] == "down", "viz_value"] *= -1
+
+    fig = go.Figure(
+        data=[
+            go.Bar(
+                name=metric,
+                x=df.name.unique(),
+                y=df.loc[df["metric"] == metric]["viz_value"],
+                hovertext=df.loc[df["metric"] == metric]["direction"],
+            )
+            for metric in df.metric.unique()
+        ]
+    )
+    # Change the bar mode
+    fig.update_layout(barmode="relative")
+
+    return fig
+
+
+def scatter(df, metric):
+    direction = df.loc[df.metric == metric]["direction"].iloc[0]
+
+    return px.scatter(
+        df.loc[df.metric == metric].sort_values("value", ascending=direction == "up"),
+        x="name",
+        y="value",
+        title=metric.replace("_", " ").capitalize() + ", direction - " + direction,
+        hover_data=["dataset", "model"],
+    )
+
+
+def visualize_results(path, output_dir=None):
     m = MetaViewer(path, filt={"type": "model"})
 
     data = []
@@ -109,30 +163,31 @@ def visualize_results(path, output_path=None, title=None):
             )
 
     df = pd.DataFrame(data)
-    df = pd.pivot_table(
+    pv = pd.pivot_table(
         df,
         values="value",
         columns=["name"],
         index=["dataset", "model", "case", "metric", "direction"],
     )
 
-    df = df.reset_index()
-    df.loc[df["dataset"].duplicated(), "dataset"] = ""
-    df.loc[df["model"].duplicated(), "model"] = ""
-    df.loc[df["case"].duplicated(), "case"] = ""
-    df.loc[df["metric"].duplicated(), "metric"] = ""
+    pv = pv.reset_index()
+    pv.loc[pv["dataset"].duplicated(), "dataset"] = ""
+    pv.loc[pv["model"].duplicated(), "model"] = ""
+    pv.loc[pv["case"].duplicated(), "case"] = ""
+    pv.loc[pv["metric"].duplicated(), "metric"] = ""
 
-    fig = go.Figure(
-        data=[
-            go.Table(
-                header=dict(values=list(df.columns), align="left"),
-                cells=dict(values=[df[col] for col in df.columns], align="left"),
-            )
-        ]
-    )
-    fig.update_layout(title=title)
+    metrics = pv.metric.unique()
 
-    if output_path is not None:
-        fig.write_image(output_path)
+    figs = dict()
+    figs["table"] = table(pv)
 
-    return fig
+    for name in metrics:
+        figs[f"scatter_{name}"] = scatter(df, name)
+
+    figs["bar"] = relative_bar(df)
+
+    for name in figs:
+        if output_dir is not None:
+            figs[name].write_image(os.path.join(output_dir, name + ".png"))
+
+    return figs

From 4f0bf73b7aa6c76c4836097d917c21af4883d7dd Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 19:41:16 +0300
Subject: [PATCH 04/19] Format floats in results table

---
 xaib/evaluation/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/xaib/evaluation/utils.py b/xaib/evaluation/utils.py
index c23834f..27da333 100644
--- a/xaib/evaluation/utils.py
+++ b/xaib/evaluation/utils.py
@@ -90,6 +90,10 @@ def wrap_case():
 
 
 def table(df):
+    for col in df:
+        if df[col].dtype == np.float64:
+            df[col] = df[col].apply(lambda x: f"{x:0.2f}")
+
     fig = go.Figure(
         data=[
             go.Table(

From a05f1a04e173ea4a7cf7e97bea44d4c29b0f16b3 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 20:21:03 +0300
Subject: [PATCH 05/19] Add new dataset and try to adapt evaluation and vis

---
 xaib/datasets/synthetic_dataset.py            | 14 +++++---
 xaib/evaluation/dataset_factory.py            | 25 +++++++++----
 .../feature_importance/feature_importance.py  | 36 +++++++++----------
 xaib/evaluation/utils.py                      |  3 +-
 4 files changed, 45 insertions(+), 33 deletions(-)

diff --git a/xaib/datasets/synthetic_dataset.py b/xaib/datasets/synthetic_dataset.py
index 27594ad..87e7a17 100644
--- a/xaib/datasets/synthetic_dataset.py
+++ b/xaib/datasets/synthetic_dataset.py
@@ -6,21 +6,25 @@
 
 
 class SyntheticDataset(cdd.SizedDataset):
-    def __init__(self, split, frac=0.8, *args, **kwargs) -> None:
+    def __init__(self, split, name=None, frac=0.8, *args, **kwargs) -> None:
         super().__init__(**kwargs)
-        self.name = 'synthetic'
+        # Useful for different synthetic datasets
+        if name is not None:
+            self.name = name
+        else:
+            self.name = "synthetic"
 
         x, y = make_classification(*args, **kwargs)
 
         train_x, test_x, train_y, test_y = train_test_split(x, y, train_size=frac)
 
-        if split == 'train':
+        if split == "train":
             self.x, self.y = train_x, train_y
-        elif split == 'test':
+        elif split == "test":
             self.x, self.y = test_x, test_y
 
     def __len__(self) -> int:
         return len(self.x)
 
     def __getitem__(self, index: int) -> Dict[str, Any]:
-        return {'item': self.x[index], 'label': self.y[index]}
+        return {"item": self.x[index], "label": self.y[index]}
diff --git a/xaib/evaluation/dataset_factory.py b/xaib/evaluation/dataset_factory.py
index 4097337..d240cc6 100644
--- a/xaib/evaluation/dataset_factory.py
+++ b/xaib/evaluation/dataset_factory.py
@@ -2,22 +2,35 @@
 from ..datasets.synthetic_dataset import SyntheticDataset
 
 
-def generate_dataset(frac: float = 0.9, **kwargs):
-    train_ds, test_ds = (SyntheticDataset('train', frac=frac, **kwargs),
-                         SyntheticDataset('test', frac=frac, **kwargs))
+def generate_dataset(name="synthetic", frac: float = 0.9, **kwargs):
+    train_ds, test_ds = (
+        SyntheticDataset("train", name=name, frac=frac, **kwargs),
+        SyntheticDataset("test", name=name, frac=frac, **kwargs),
+    )
     return train_ds, test_ds
 
 
 class DatasetFactory(Factory):
     def __init__(self) -> None:
         super().__init__()
-        self._constructors['synthetic'] = lambda: generate_dataset(
-            n_samples=1000,
+        self._constructors["synthetic"] = lambda: generate_dataset(
+            n_samples=100,
             n_features=10,
             random_state=0,
             n_informative=5,
             n_redundant=1,
             n_repeated=0,
             n_clusters_per_class=1,
-            frac=0.9
+            frac=0.8,
+        )
+        self._constructors["synthetic_noisy"] = lambda: generate_dataset(
+            name="synthetic_noisy",
+            n_samples=100,
+            n_features=14,
+            random_state=0,
+            n_informative=7,
+            n_redundant=5,
+            n_repeated=2,
+            n_clusters_per_class=2,
+            frac=0.8,
         )
diff --git a/xaib/evaluation/feature_importance/feature_importance.py b/xaib/evaluation/feature_importance/feature_importance.py
index ba75e77..9d01588 100644
--- a/xaib/evaluation/feature_importance/feature_importance.py
+++ b/xaib/evaluation/feature_importance/feature_importance.py
@@ -6,6 +6,7 @@
 from cascade.models import ModelRepo
 
 from xaib.evaluation.feature_importance import ExplainerFactory, ExperimentFactory
+from xaib.evaluation import ModelFactory, DatasetFactory
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 BASE_DIR = os.path.dirname(SCRIPT_DIR)
@@ -16,33 +17,28 @@
 from xaib.evaluation.model_factory import SkWrapper
 
 
-REPO_PATH = os.path.join(os.path.dirname(BASE_DIR), 'results', 'feature_importance')
+REPO_PATH = os.path.join(os.path.dirname(BASE_DIR), "results", "feature_importance")
 BS = 5
 
 
 # Overwrite previous run
 ModelRepo(REPO_PATH, overwrite=True)
 
+for dataset in ["synthetic_noisy", "synthetic"]:
+    for model in ["svm"]:
+        train_ds, test_ds = DatasetFactory().get(dataset)
+        print(train_ds.get_meta())
 
-train_ds = cdd.Pickler(os.path.join(SCRIPT_DIR, 'train_ds')).ds()
-test_ds = cdd.Pickler(os.path.join(SCRIPT_DIR, 'test_ds')).ds()
+        model = ModelFactory(train_ds, test_ds).get(model)
+        print(model.get_meta())
 
-model = SkWrapper(name='svm')
-model.load(os.path.join(SCRIPT_DIR, 'svm'))
+        explainers = ExplainerFactory(train_ds, model).get("all")
+        experiment_factory = ExperimentFactory(
+            REPO_PATH, explainers, test_ds, model, BS
+        )
 
-explainers = ExplainerFactory(train_ds, model).get('all')
+        experiments = experiment_factory.get("all")
+        for name in experiments:
+            experiments[name]()
 
-experiment_factory = ExperimentFactory(
-    REPO_PATH,
-    explainers,
-    test_ds,
-    model,
-    BS
-)
-
-experiments = experiment_factory.get('all')
-for name in experiments:
-    experiments[name]()
-
-
-visualize_results(REPO_PATH, os.path.join(REPO_PATH, 'results.png'))
+visualize_results(REPO_PATH, REPO_PATH)
diff --git a/xaib/evaluation/utils.py b/xaib/evaluation/utils.py
index 27da333..16177f0 100644
--- a/xaib/evaluation/utils.py
+++ b/xaib/evaluation/utils.py
@@ -180,11 +180,10 @@ def visualize_results(path, output_dir=None):
     pv.loc[pv["case"].duplicated(), "case"] = ""
     pv.loc[pv["metric"].duplicated(), "metric"] = ""
 
-    metrics = pv.metric.unique()
-
     figs = dict()
     figs["table"] = table(pv)
 
+    metrics = df.metric.unique()
     for name in metrics:
         figs[f"scatter_{name}"] = scatter(df, name)
 

From a10dfb6c0707ae96b9b78b298310b6f9e61c6af6 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 20:21:11 +0300
Subject: [PATCH 06/19] Format conf.py

---
 xaib/docs/source/conf.py | 48 +++++++++++++++++++---------------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/xaib/docs/source/conf.py b/xaib/docs/source/conf.py
index 69431c5..730def4 100644
--- a/xaib/docs/source/conf.py
+++ b/xaib/docs/source/conf.py
@@ -12,15 +12,16 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../../..'))
+
+sys.path.insert(0, os.path.abspath("../../.."))
 
 import xaib
 
 # -- Project information -----------------------------------------------------
 
-project = 'xai-benchmark'
-copyright = '2023, Ilia Moiseev'
-author = 'Ilia Moiseev'
+project = "xai-benchmark"
+copyright = "2023, Ilia Moiseev"
+author = "Ilia Moiseev"
 
 # The full version, including alpha/beta/rc tags
 release = xaib.__version__
@@ -32,22 +33,19 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.napoleon',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
     "sphinx.ext.todo",
     "sphinx.ext.viewcode",
     "sphinx.ext.mathjax",
     "sphinx.ext.extlinks",
-    "sphinx_copybutton"
+    "sphinx_copybutton",
 ]
 
-autodoc_default_options = {
-    'special-members': '__init__',
-    'undoc-members': False
-}
+autodoc_default_options = {"special-members": "__init__", "undoc-members": False}
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -60,7 +58,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_material'
+html_theme = "sphinx_material"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -69,18 +67,18 @@
 
 
 html_theme_options = {
-    'base_url': 'http://oxid15.github.io/xai-benchmark/',
-    'repo_url': 'https://github.com/Oxid15/xai-benchmark/',
-    'repo_name': 'xai-benchmark',
-    'repo_type': 'github',
-    'globaltoc_depth': 2,
-    'globaltoc_collapse': True,
-    'color_primary': 'white',
-    'color_accent': 'red',
-    'html_minify': True,
-    'css_minify': True,
-    'nav_title': 'XAIB - Open and extensible benchmark for XAI methods',
-    'logo_icon': '&#x8cea' #'&#x7d30&#x90e8'
+    "base_url": "http://oxid15.github.io/xai-benchmark/",
+    "repo_url": "https://github.com/Oxid15/xai-benchmark/",
+    "repo_name": "xai-benchmark",
+    "repo_type": "github",
+    "globaltoc_depth": 2,
+    "globaltoc_collapse": True,
+    "color_primary": "white",
+    "color_accent": "red",
+    "html_minify": True,
+    "css_minify": True,
+    "nav_title": "XAIB - Open and extensible benchmark for XAI methods",
+    "logo_icon": "&#x8cea",  #'&#x7d30&#x90e8'
 }
 
 html_sidebars = {

From 2770e45bffd49fe4bd19c8eab054643daa901b63 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 20:34:43 +0300
Subject: [PATCH 07/19] Make bar average across datasets and models

---
 xaib/evaluation/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/xaib/evaluation/utils.py b/xaib/evaluation/utils.py
index 16177f0..d576473 100644
--- a/xaib/evaluation/utils.py
+++ b/xaib/evaluation/utils.py
@@ -110,6 +110,8 @@ def table(df):
 def relative_bar(df):
     metrics = df.metric.unique()
 
+    df = df.groupby(["name", "metric", "direction"], as_index=False).agg("mean")
+
     df["viz_value"] = df["value"]
     for metric in metrics:
         df.loc[df["metric"] == metric, "viz_value"] /= df.loc[

From 8a8fe643316586892953b77dcafcc2465e250734 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 20:35:02 +0300
Subject: [PATCH 08/19] Fix imports

---
 xaib/evaluation/feature_importance/feature_importance.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/xaib/evaluation/feature_importance/feature_importance.py b/xaib/evaluation/feature_importance/feature_importance.py
index 9d01588..3901621 100644
--- a/xaib/evaluation/feature_importance/feature_importance.py
+++ b/xaib/evaluation/feature_importance/feature_importance.py
@@ -1,12 +1,9 @@
 import os
 import sys
 
-from cascade import data as cdd
-from cascade.utils.sk_model import SkModel
 from cascade.models import ModelRepo
-
-from xaib.evaluation.feature_importance import ExplainerFactory, ExperimentFactory
-from xaib.evaluation import ModelFactory, DatasetFactory
+from xaib.evaluation import DatasetFactory, ModelFactory
+from xaib.evaluation.feature_importance import ExperimentFactory, ExplainerFactory
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 BASE_DIR = os.path.dirname(SCRIPT_DIR)
@@ -14,8 +11,6 @@
 sys.path.append(PROJECT_DIR)
 
 from xaib.evaluation.utils import visualize_results
-from xaib.evaluation.model_factory import SkWrapper
-
 
 REPO_PATH = os.path.join(os.path.dirname(BASE_DIR), "results", "feature_importance")
 BS = 5

From 1301d4c6696ae7712a2e0d5d839454329f30bd8f Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 20:41:32 +0300
Subject: [PATCH 09/19] Make table skip repeating values

---
 xaib/evaluation/utils.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/xaib/evaluation/utils.py b/xaib/evaluation/utils.py
index d576473..13a86c4 100644
--- a/xaib/evaluation/utils.py
+++ b/xaib/evaluation/utils.py
@@ -177,10 +177,14 @@ def visualize_results(path, output_dir=None):
     )
 
     pv = pv.reset_index()
-    pv.loc[pv["dataset"].duplicated(), "dataset"] = ""
-    pv.loc[pv["model"].duplicated(), "model"] = ""
-    pv.loc[pv["case"].duplicated(), "case"] = ""
-    pv.loc[pv["metric"].duplicated(), "metric"] = ""
+
+    for col in ["dataset", "model", "case", "metric"]:
+        prev_val = ""
+        for i in range(len(pv[col])):
+            val = pv.loc[i][col]
+            if val == prev_val:
+                pv.loc[i, col] = ""
+            prev_val = val
 
     figs = dict()
     figs["table"] = table(pv)

From 71fe4fe9d82c825009607875f088ede0157c04d8 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 20:43:20 +0300
Subject: [PATCH 10/19] Change synth datasets params

---
 xaib/evaluation/dataset_factory.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/xaib/evaluation/dataset_factory.py b/xaib/evaluation/dataset_factory.py
index d240cc6..469b1d2 100644
--- a/xaib/evaluation/dataset_factory.py
+++ b/xaib/evaluation/dataset_factory.py
@@ -2,7 +2,7 @@
 from ..datasets.synthetic_dataset import SyntheticDataset
 
 
-def generate_dataset(name="synthetic", frac: float = 0.9, **kwargs):
+def generate_dataset(name=None, frac: float = 0.9, **kwargs):
     train_ds, test_ds = (
         SyntheticDataset("train", name=name, frac=frac, **kwargs),
         SyntheticDataset("test", name=name, frac=frac, **kwargs),
@@ -15,10 +15,10 @@ def __init__(self) -> None:
         super().__init__()
         self._constructors["synthetic"] = lambda: generate_dataset(
             n_samples=100,
-            n_features=10,
+            n_features=14,
             random_state=0,
-            n_informative=5,
-            n_redundant=1,
+            n_informative=14,
+            n_redundant=0,
             n_repeated=0,
             n_clusters_per_class=1,
             frac=0.8,

From 74fb5d18aeeedf4bd07b8676782f11aabaa89cbc Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sat, 1 Apr 2023 20:46:23 +0300
Subject: [PATCH 11/19] New evaluation everywhere

---
 xaib/evaluation/example_selection/evaluate.sh |  2 -
 .../example_selection/example_selection.py    | 40 +++++++++----------
 .../example_selection/make_dataset.py         | 16 --------
 .../example_selection/train_knn_model.py      | 18 ---------
 .../evaluation/feature_importance/evaluate.sh |  2 -
 .../feature_importance/make_dataset.py        | 16 --------
 .../feature_importance/train_model.py         | 24 -----------
 7 files changed, 20 insertions(+), 98 deletions(-)
 delete mode 100644 xaib/evaluation/example_selection/make_dataset.py
 delete mode 100644 xaib/evaluation/example_selection/train_knn_model.py
 delete mode 100644 xaib/evaluation/feature_importance/make_dataset.py
 delete mode 100644 xaib/evaluation/feature_importance/train_model.py

diff --git a/xaib/evaluation/example_selection/evaluate.sh b/xaib/evaluation/example_selection/evaluate.sh
index e926080..4c303a5 100755
--- a/xaib/evaluation/example_selection/evaluate.sh
+++ b/xaib/evaluation/example_selection/evaluate.sh
@@ -1,4 +1,2 @@
 #!/bin/sh
-python3 make_dataset.py
-python train_knn_model.py
 python example_selection.py
diff --git a/xaib/evaluation/example_selection/example_selection.py b/xaib/evaluation/example_selection/example_selection.py
index 497927f..645f994 100644
--- a/xaib/evaluation/example_selection/example_selection.py
+++ b/xaib/evaluation/example_selection/example_selection.py
@@ -5,12 +5,15 @@
 from cascade import data as cdd
 from cascade.utils.sk_model import SkModel
 
+from xaib.evaluation import DatasetFactory, ModelFactory
 from xaib.evaluation.example_selection import ExplainerFactory, ExperimentFactory
 
 
 SCRIPT_DIR = os.path.dirname(__file__)
 # xaib/results/...
-REPO_PATH = os.path.join(os.path.dirname(os.path.dirname(SCRIPT_DIR)), 'results', 'example_selection')
+REPO_PATH = os.path.join(
+    os.path.dirname(os.path.dirname(SCRIPT_DIR)), "results", "example_selection"
+)
 
 sys.path.append(os.path.abspath(os.path.dirname(SCRIPT_DIR)))
 from utils import visualize_results, WrapperModel
@@ -22,29 +25,26 @@ def __init__(self, *args, blocks=None, name=None, **kwargs) -> None:
         self.name = name
 
 
-# Overwrite previous run
-ModelRepo(REPO_PATH, overwrite=True)
-
 BS = 5
 
-train_ds = cdd.Pickler(os.path.join(SCRIPT_DIR, 'train_ds')).ds()
-test_ds = cdd.Pickler(os.path.join(SCRIPT_DIR, 'test_ds')).ds()
+# Overwrite previous run
+ModelRepo(REPO_PATH, overwrite=True)
 
-model = SkWrapper(name='knn')
-model.load(os.path.join(SCRIPT_DIR, 'knn'))
+for dataset in ["synthetic_noisy", "synthetic"]:
+    for model in ["knn"]:
+        train_ds, test_ds = DatasetFactory().get(dataset)
+        print(train_ds.get_meta())
 
-explainers = ExplainerFactory(train_ds, model).get('all')
+        model = ModelFactory(train_ds, test_ds).get(model)
+        print(model.get_meta())
 
-experiment_factory = ExperimentFactory(
-    REPO_PATH,
-    explainers,
-    test_ds,
-    model,
-    BS
-)
+        explainers = ExplainerFactory(train_ds, model).get("all")
+        experiment_factory = ExperimentFactory(
+            REPO_PATH, explainers, test_ds, model, BS
+        )
 
-experiments = experiment_factory.get('all')
-for name in experiments:
-    experiments[name]()
+        experiments = experiment_factory.get("all")
+        for name in experiments:
+            experiments[name]()
 
-visualize_results(REPO_PATH, os.path.join(REPO_PATH, 'results.png'))
+visualize_results(REPO_PATH, REPO_PATH)
diff --git a/xaib/evaluation/example_selection/make_dataset.py b/xaib/evaluation/example_selection/make_dataset.py
deleted file mode 100644
index 13ee9c2..0000000
--- a/xaib/evaluation/example_selection/make_dataset.py
+++ /dev/null
@@ -1,16 +0,0 @@
-import os
-import sys
-from pprint import pprint
-from cascade import data as cdd
-from xaib.evaluation import DatasetFactory
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-
-
-train_ds, test_ds = DatasetFactory().get('synthetic')
-
-cdd.Pickler(os.path.join(SCRIPT_DIR, 'train_ds'), train_ds)
-cdd.Pickler(os.path.join(SCRIPT_DIR, 'test_ds'), test_ds)
-
-pprint(train_ds.get_meta())
-pprint(test_ds.get_meta())
diff --git a/xaib/evaluation/example_selection/train_knn_model.py b/xaib/evaluation/example_selection/train_knn_model.py
deleted file mode 100644
index d68fade..0000000
--- a/xaib/evaluation/example_selection/train_knn_model.py
+++ /dev/null
@@ -1,18 +0,0 @@
-import os
-
-from xaib.datasets.synthetic_dataset import SyntheticDataset
-from xaib.evaluation import ModelFactory
-from cascade import data as cdd
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-
-
-N_NEIGHBORS = 3
-
-
-train_ds = cdd.Pickler(os.path.join(SCRIPT_DIR, 'train_ds'))
-test_ds = cdd.Pickler(os.path.join(SCRIPT_DIR, 'train_ds'))
-
-model = ModelFactory(train_ds, test_ds).get('knn')
-print(model.get_meta())
-model.save(os.path.join(SCRIPT_DIR, 'knn'))
diff --git a/xaib/evaluation/feature_importance/evaluate.sh b/xaib/evaluation/feature_importance/evaluate.sh
index dff4bac..db80548 100755
--- a/xaib/evaluation/feature_importance/evaluate.sh
+++ b/xaib/evaluation/feature_importance/evaluate.sh
@@ -1,4 +1,2 @@
 #!/bin/sh
-python3 make_dataset.py
-python train_model.py
 python feature_importance.py
diff --git a/xaib/evaluation/feature_importance/make_dataset.py b/xaib/evaluation/feature_importance/make_dataset.py
deleted file mode 100644
index 13ee9c2..0000000
--- a/xaib/evaluation/feature_importance/make_dataset.py
+++ /dev/null
@@ -1,16 +0,0 @@
-import os
-import sys
-from pprint import pprint
-from cascade import data as cdd
-from xaib.evaluation import DatasetFactory
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-
-
-train_ds, test_ds = DatasetFactory().get('synthetic')
-
-cdd.Pickler(os.path.join(SCRIPT_DIR, 'train_ds'), train_ds)
-cdd.Pickler(os.path.join(SCRIPT_DIR, 'test_ds'), test_ds)
-
-pprint(train_ds.get_meta())
-pprint(test_ds.get_meta())
diff --git a/xaib/evaluation/feature_importance/train_model.py b/xaib/evaluation/feature_importance/train_model.py
deleted file mode 100644
index 9e4c933..0000000
--- a/xaib/evaluation/feature_importance/train_model.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import os
-import sys
-
-import numpy as np
-from cascade import data as cdd
-from cascade import utils as cdu
-from sklearn.svm import SVC
-from sklearn.metrics import f1_score
-
-from xaib.evaluation import ModelFactory
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-PROJECT_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR))
-sys.path.append(PROJECT_DIR)
-from datasets import SyntheticDataset
-
-
-train_ds = cdd.Pickler(os.path.join(SCRIPT_DIR, 'train_ds'))
-test_ds = cdd.Pickler(os.path.join(SCRIPT_DIR, 'train_ds'))
-
-model = ModelFactory(train_ds, test_ds).get('svm')
-
-print(model.get_meta())
-model.save(os.path.join(SCRIPT_DIR, 'svm'))

From 1c6cba624587c64afdb8694e53803b43dfa46285 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sun, 2 Apr 2023 11:23:27 +0300
Subject: [PATCH 12/19] Housekeeping

---
 xaib/__init__.py                              |  8 +--
 xaib/base/__init__.py                         |  2 +-
 xaib/base/base.py                             | 61 ++++++++++++-------
 xaib/cases/example_selection/__init__.py      |  2 +-
 .../example_selection/continuity_case.py      | 11 ++--
 .../example_selection/contrastivity_case.py   |  9 ++-
 .../example_selection/correctness_case.py     | 13 ++--
 .../covariate_complexity_case.py              |  7 ++-
 xaib/cases/feature_importance/__init__.py     |  6 +-
 .../feature_importance/coherence_case.py      |  7 ++-
 .../feature_importance/compactness_case.py    |  7 ++-
 .../feature_importance/continuity_case.py     | 11 ++--
 .../feature_importance/contrastivity_case.py  |  7 ++-
 .../feature_importance/correctness_case.py    | 13 ++--
 .../covariate_complexity_case.py              |  7 ++-
 xaib/datasets/synthetic_dataset.py            |  4 +-
 xaib/docs/embed_plots.py                      | 16 +++--
 xaib/docs/source/results.rst                  |  8 +++
 xaib/evaluation/example_selection/__init__.py |  2 +-
 .../example_selection/case_factory.py         | 33 +++++-----
 .../example_selection/example_selection.py    |  8 +--
 .../example_selection/experiment_factory.py   | 18 +++---
 .../example_selection/explainer_factory.py    | 12 ++--
 .../evaluation/feature_importance/__init__.py |  2 +-
 .../feature_importance/case_factory.py        | 33 +++++-----
 .../feature_importance/experiment_factory.py  | 41 +++++++------
 .../feature_importance/explainer_factory.py   | 12 ++--
 xaib/evaluation/model_factory.py              | 24 ++++----
 xaib/evaluation/utils.py                      | 35 ++++++++---
 .../example_selection/constant_explainer.py   |  1 +
 .../example_selection/knn_explainer.py        |  5 +-
 .../example_selection/random_explainer.py     |  8 ++-
 .../feature_importance/constant_explainer.py  |  7 ++-
 .../feature_importance/lime_explainer.py      | 16 +++--
 .../feature_importance/random_explainer.py    |  4 +-
 .../feature_importance/shap_explainer.py      |  2 +-
 xaib/metrics/example_selection/__init__.py    |  4 +-
 .../example_selection/covariate_regularity.py | 25 +++++---
 .../parameter_randomization_check.py          | 24 +++++---
 .../example_selection/small_noise_check.py    | 29 ++++-----
 .../target_discriminativeness.py              | 20 +++---
 xaib/metrics/feature_importance/__init__.py   |  6 +-
 .../covariate_regularity.py                   | 22 ++++---
 .../feature_importance/label_difference.py    | 30 +++++----
 .../feature_importance/other_disagreement.py  | 22 ++++---
 .../parameter_randomization_check.py          |  8 ++-
 .../feature_importance/small_noise_check.py   | 30 +++++----
 xaib/metrics/feature_importance/sparsity.py   | 15 ++---
 xaib/tests/test_feature_importance_shap.py    | 15 -----
 xaib/utils/__init__.py                        | 10 ++-
 xaib/utils/utils.py                           |  5 +-
 51 files changed, 408 insertions(+), 319 deletions(-)
 delete mode 100644 xaib/tests/test_feature_importance_shap.py

diff --git a/xaib/__init__.py b/xaib/__init__.py
index 13f723e..2849723 100644
--- a/xaib/__init__.py
+++ b/xaib/__init__.py
@@ -1,6 +1,6 @@
-__version__ = '0.2.0-alpha'
-__author__ = 'Ilia Moiseev'
-__author_email__ = 'ilia.moiseev.5@yandex.ru'
+__version__ = "0.2.0-alpha"
+__author__ = "Ilia Moiseev"
+__author_email__ = "ilia.moiseev.5@yandex.ru"
 
 
-from .base import Dataset, Model, Explainer, Metric, Case, Factory
+from .base import Case, Dataset, Explainer, Factory, Metric, Model
diff --git a/xaib/base/__init__.py b/xaib/base/__init__.py
index 6e5b653..f4d2136 100644
--- a/xaib/base/__init__.py
+++ b/xaib/base/__init__.py
@@ -1 +1 @@
-from .base import Dataset, Model, Explainer, Metric, Case, Factory
+from .base import Case, Dataset, Explainer, Factory, Metric, Model
diff --git a/xaib/base/base.py b/xaib/base/base.py
index 1e38b02..43b1846 100644
--- a/xaib/base/base.py
+++ b/xaib/base/base.py
@@ -8,6 +8,7 @@ class Dataset(cdd.Dataset):
     Dataset is a wrapper around any collection of items to put in the ML model
     for inference
     """
+
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -21,6 +22,7 @@ class Model(cdm.Model):
     Model is a wrapper around any inference of ML or other solution in the form y = f(x)
     it implements method `predict` that given certain data x returns the response y
     """
+
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -34,6 +36,7 @@ class Explainer(cdm.Model):
     Explainer is a special kind of Model e = g(f, x) that accepts another Model and data as input
     and also returns a response e - an explanation
     """
+
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -47,6 +50,7 @@ class Metric(cdm.Model):
     Metric is an entity which accepts Explainer, Model and Dataset and outputs a metric
     corresponding to the quality of Explainer v = m(g, f, x)
     """
+
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -54,25 +58,30 @@ def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None
         self._ds = ds
         self._model = model
 
-    def compute(self, expl: Explainer, batch_size: int = 1, expl_kwargs: Union[Dict[Any, Any], None] = None) -> Any:
+    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> Any:
         raise NotImplementedError()
 
     def evaluate(
-            self,
-            name: str,
-            expl: Explainer,
-            batch_size: int = 1,
-            expl_kwargs: Union[Dict[Any, Any], None] = None,
-            **kwargs: Any
+        self,
+        name: str,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+        **kwargs: Any
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
         value = self.compute(expl, batch_size=batch_size, **expl_kwargs, **kwargs)
 
-        self.params['name'] = name
-        self.params['direction'] = self.direction
-        self.params['dataset'] = self._ds.name
-        self.params['model'] = self._model.name
+        self.params["name"] = name
+        self.params["direction"] = self.direction
+        self.params["dataset"] = self._ds.name
+        self.params["model"] = self._model.name
         self.metrics[self.name] = value
 
 
@@ -81,6 +90,7 @@ class Case(cdm.Model):
     Case is a collection of Metrics which represent some
     high-level property of an Explainer
     """
+
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -89,11 +99,17 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
     def add_metric(self, name: str, metric: Metric) -> None:
         self._metric_objs[name] = metric
 
-    def evaluate(self, name: str, expl: Explainer, metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None, **kwargs: Any) -> None:
+    def evaluate(
+        self,
+        name: str,
+        expl: Explainer,
+        metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None,
+        **kwargs: Any
+    ) -> None:
         if metrics_kwargs is None:
             metrics_kwargs = {name: {} for _ in self._metric_objs}
 
-        self.params['metric_params'] = dict()
+        self.params["metric_params"] = dict()
         for m_name in self._metric_objs:
             mkwargs = {}
             if m_name in metrics_kwargs:
@@ -101,9 +117,9 @@ def evaluate(self, name: str, expl: Explainer, metrics_kwargs: Union[Dict[str, D
 
             self._metric_objs[m_name].evaluate(name, expl, **mkwargs, **kwargs)
 
-            self.params['case'] = self.name
+            self.params["case"] = self.name
 
-            self.params['metric_params'][m_name] = (self._metric_objs[m_name].params)
+            self.params["metric_params"][m_name] = self._metric_objs[m_name].params
             self.metrics.update(self._metric_objs[m_name].metrics)
 
 
@@ -112,15 +128,13 @@ class Factory:
     Collection of constructors to build and return objects
     from predefined hardcoded or added dynamically constructors
     """
+
     def __init__(self) -> None:
         self._constructors = dict()
         self._constructors_kwargs = dict()
 
     def _get_all(self) -> Dict[str, Any]:
-        return {
-                name: self._get(name)
-                for name in self._constructors
-            }
+        return {name: self._get(name) for name in self._constructors}
 
     def _get(self, name: str) -> Any:
         if name not in self._constructors_kwargs:
@@ -132,10 +146,15 @@ def _get(self, name: str) -> Any:
         return constructor(**kwargs)
 
     def get(self, name: str) -> Union[Dict[str, Any], Any]:
-        if name == 'all':
+        if name == "all":
             return self._get_all()
         return self._get(name)
 
-    def add(self, name: str, constructor: Callable[[Any], Any], constr_kwargs: Union[Any, None] = None) -> None:
+    def add(
+        self,
+        name: str,
+        constructor: Callable[[Any], Any],
+        constr_kwargs: Union[Any, None] = None,
+    ) -> None:
         self._constructors[name] = constructor
         self._constructors_kwargs[name] = constr_kwargs
diff --git a/xaib/cases/example_selection/__init__.py b/xaib/cases/example_selection/__init__.py
index 49f4231..b04c393 100644
--- a/xaib/cases/example_selection/__init__.py
+++ b/xaib/cases/example_selection/__init__.py
@@ -1,4 +1,4 @@
 from .continuity_case import ContinuityCase
 from .contrastivity_case import ContrastivityCase
-from .covariate_complexity_case import CovariateComplexityCase
 from .correctness_case import CorrectnessCase
+from .covariate_complexity_case import CovariateComplexityCase
diff --git a/xaib/cases/example_selection/continuity_case.py b/xaib/cases/example_selection/continuity_case.py
index d906a6b..aeb4547 100644
--- a/xaib/cases/example_selection/continuity_case.py
+++ b/xaib/cases/example_selection/continuity_case.py
@@ -1,10 +1,13 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.example_selection import SmallNoiseCheck
 
 
 class ContinuityCase(Case):
-    def __init__(self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+    def __init__(
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'continuity'
-        self._metric_objs['small_noise_check'] = SmallNoiseCheck(ds, noisy_ds, model)
+        self.name = "continuity"
+        self._metric_objs["small_noise_check"] = SmallNoiseCheck(ds, noisy_ds, model)
diff --git a/xaib/cases/example_selection/contrastivity_case.py b/xaib/cases/example_selection/contrastivity_case.py
index 6c29443..e41802f 100644
--- a/xaib/cases/example_selection/contrastivity_case.py
+++ b/xaib/cases/example_selection/contrastivity_case.py
@@ -1,10 +1,13 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.example_selection import TargetDiscriminativeness
 
 
 class ContrastivityCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'contrastivity'
-        self._metric_objs['target_discriminativeness'] = TargetDiscriminativeness(ds, model)
+        self.name = "contrastivity"
+        self._metric_objs["target_discriminativeness"] = TargetDiscriminativeness(
+            ds, model
+        )
diff --git a/xaib/cases/example_selection/correctness_case.py b/xaib/cases/example_selection/correctness_case.py
index c4dd3d9..12e0243 100644
--- a/xaib/cases/example_selection/correctness_case.py
+++ b/xaib/cases/example_selection/correctness_case.py
@@ -1,10 +1,15 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.example_selection import ParameterRandomizationCheck
 
 
 class CorrectnessCase(Case):
-    def __init__(self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any) -> None:
+    def __init__(
+        self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'correctness'
-        self._metric_objs['parameter_randomization_check'] = ParameterRandomizationCheck(ds, model, noisy_model)
+        self.name = "correctness"
+        self._metric_objs[
+            "parameter_randomization_check"
+        ] = ParameterRandomizationCheck(ds, model, noisy_model)
diff --git a/xaib/cases/example_selection/covariate_complexity_case.py b/xaib/cases/example_selection/covariate_complexity_case.py
index ccc88bc..bee82cf 100644
--- a/xaib/cases/example_selection/covariate_complexity_case.py
+++ b/xaib/cases/example_selection/covariate_complexity_case.py
@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.example_selection import CovariateRegularity
 
 
 class CovariateComplexityCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'covariate_complexity'
-        self._metric_objs['covariate_regularity'] = CovariateRegularity(ds, model)
+        self.name = "covariate_complexity"
+        self._metric_objs["covariate_regularity"] = CovariateRegularity(ds, model)
diff --git a/xaib/cases/feature_importance/__init__.py b/xaib/cases/feature_importance/__init__.py
index 1975d95..3c6a47f 100644
--- a/xaib/cases/feature_importance/__init__.py
+++ b/xaib/cases/feature_importance/__init__.py
@@ -1,6 +1,6 @@
-from .correctness_case import CorrectnessCase
+from .coherence_case import CoherenceCase
+from .compactness_case import CompactnessCase
 from .continuity_case import ContinuityCase
 from .contrastivity_case import ContrastivityCase
-from .coherence_case import CoherenceCase
+from .correctness_case import CorrectnessCase
 from .covariate_complexity_case import CovariateComplexityCase
-from .compactness_case import CompactnessCase
diff --git a/xaib/cases/feature_importance/coherence_case.py b/xaib/cases/feature_importance/coherence_case.py
index a43f89f..29656ea 100644
--- a/xaib/cases/feature_importance/coherence_case.py
+++ b/xaib/cases/feature_importance/coherence_case.py
@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import OtherDisagreement
 
 
 class CoherenceCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'coherence'
-        self._metric_objs['other_disagreement'] = OtherDisagreement(ds, model)
+        self.name = "coherence"
+        self._metric_objs["other_disagreement"] = OtherDisagreement(ds, model)
diff --git a/xaib/cases/feature_importance/compactness_case.py b/xaib/cases/feature_importance/compactness_case.py
index aa2242d..871e324 100644
--- a/xaib/cases/feature_importance/compactness_case.py
+++ b/xaib/cases/feature_importance/compactness_case.py
@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import Sparsity
 
 
 class CompactnessCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'compactness'
-        self._metric_objs['sparsity'] = Sparsity(ds, model)
+        self.name = "compactness"
+        self._metric_objs["sparsity"] = Sparsity(ds, model)
diff --git a/xaib/cases/feature_importance/continuity_case.py b/xaib/cases/feature_importance/continuity_case.py
index de2f6a5..08db555 100644
--- a/xaib/cases/feature_importance/continuity_case.py
+++ b/xaib/cases/feature_importance/continuity_case.py
@@ -1,10 +1,13 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import SmallNoiseCheck
 
 
 class ContinuityCase(Case):
-    def __init__(self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+    def __init__(
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'continuity'
-        self._metric_objs['small_noise_check'] = SmallNoiseCheck(ds, noisy_ds, model)
+        self.name = "continuity"
+        self._metric_objs["small_noise_check"] = SmallNoiseCheck(ds, noisy_ds, model)
diff --git a/xaib/cases/feature_importance/contrastivity_case.py b/xaib/cases/feature_importance/contrastivity_case.py
index 4c34ebf..e4e1f60 100644
--- a/xaib/cases/feature_importance/contrastivity_case.py
+++ b/xaib/cases/feature_importance/contrastivity_case.py
@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import LabelDifference
 
 
 class ContrastivityCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'contrastivity'
-        self._metric_objs['label_difference'] = LabelDifference(ds, model)
+        self.name = "contrastivity"
+        self._metric_objs["label_difference"] = LabelDifference(ds, model)
diff --git a/xaib/cases/feature_importance/correctness_case.py b/xaib/cases/feature_importance/correctness_case.py
index 72501c8..17260e5 100644
--- a/xaib/cases/feature_importance/correctness_case.py
+++ b/xaib/cases/feature_importance/correctness_case.py
@@ -1,10 +1,15 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import ParameterRandomizationCheck
 
 
 class CorrectnessCase(Case):
-    def __init__(self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any) -> None:
+    def __init__(
+        self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'correctness'
-        self._metric_objs['parameter_randomization_check'] = ParameterRandomizationCheck(ds, model, noisy_model)
+        self.name = "correctness"
+        self._metric_objs[
+            "parameter_randomization_check"
+        ] = ParameterRandomizationCheck(ds, model, noisy_model)
diff --git a/xaib/cases/feature_importance/covariate_complexity_case.py b/xaib/cases/feature_importance/covariate_complexity_case.py
index 97767b2..ccaa293 100644
--- a/xaib/cases/feature_importance/covariate_complexity_case.py
+++ b/xaib/cases/feature_importance/covariate_complexity_case.py
@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import CovariateRegularity
 
 
 class CovariateComplexityCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'covariate_complexity'
-        self._metric_objs['covariate_regularity'] = CovariateRegularity(ds, model)
+        self.name = "covariate_complexity"
+        self._metric_objs["covariate_regularity"] = CovariateRegularity(ds, model)
diff --git a/xaib/datasets/synthetic_dataset.py b/xaib/datasets/synthetic_dataset.py
index 87e7a17..d0a372e 100644
--- a/xaib/datasets/synthetic_dataset.py
+++ b/xaib/datasets/synthetic_dataset.py
@@ -1,8 +1,8 @@
-from typing import Dict, Any
+from typing import Any, Dict
 
+from cascade import data as cdd
 from sklearn.datasets import make_classification
 from sklearn.model_selection import train_test_split
-from cascade import data as cdd
 
 
 class SyntheticDataset(cdd.SizedDataset):
diff --git a/xaib/docs/embed_plots.py b/xaib/docs/embed_plots.py
index 149b3ca..601c9c1 100644
--- a/xaib/docs/embed_plots.py
+++ b/xaib/docs/embed_plots.py
@@ -11,22 +11,28 @@
 RESULTS = ("feature_importance", "example_selection")
 
 REPO_PATHS = [os.path.join(BASE_DIR, "xaib", "results", name) for name in RESULTS]
-INDEX_PATH = os.path.join(BASE_DIR, "xaib", "docs", "build", "results.html")
+INDEX_PATH = os.path.join(BASE_DIR, "xaib", "docs", "build", "results")
 
 for name, path in zip(RESULTS, REPO_PATHS):
     figs = visualize_results(path)
 
-    for name in figs:
-        htmlfig = to_html(figs[name], include_plotlyjs="cdn")
+    for fig in figs:
+        htmlfig = to_html(fig["fig"], include_plotlyjs="cdn")
 
         htmlfig = BeautifulSoup(htmlfig, "html.parser")
 
-        with open(INDEX_PATH, "r", encoding="UTF-8") as f:
+        with open(os.path.join(INDEX_PATH, name + ".html"), "r", encoding="UTF-8") as f:
             html = "\n".join(f.readlines())
 
         soup = BeautifulSoup(html, "html.parser")
 
+        soup.findAll("footer", {"class": "md-footer"})[0].insert_before(
+            BeautifulSoup(
+                f'<article class="md-content__inner md-typeset" role="main"><p>{fig["desc"]}</p></article>',
+                "html.parser",
+            )
+        )
         soup.findAll("footer", {"class": "md-footer"})[0].insert_before(htmlfig)
 
-        with open(INDEX_PATH, "w", encoding="UTF-8") as f:
+        with open(os.path.join(INDEX_PATH, name + ".html"), "w", encoding="UTF-8") as f:
             f.write(soup.prettify())
diff --git a/xaib/docs/source/results.rst b/xaib/docs/source/results.rst
index c818ac4..32265ec 100644
--- a/xaib/docs/source/results.rst
+++ b/xaib/docs/source/results.rst
@@ -1,2 +1,10 @@
 Evaluation results
 ==================
+
+Evaluation results are available for each explanation type separately
+
+.. toctree::
+    :maxdepth: 1
+
+    results/feature_importance
+    results/example_selection
\ No newline at end of file
diff --git a/xaib/evaluation/example_selection/__init__.py b/xaib/evaluation/example_selection/__init__.py
index 6e56376..520237c 100644
--- a/xaib/evaluation/example_selection/__init__.py
+++ b/xaib/evaluation/example_selection/__init__.py
@@ -1,3 +1,3 @@
-from .explainer_factory import ExplainerFactory
 from .case_factory import CaseFactory
 from .experiment_factory import ExperimentFactory
+from .explainer_factory import ExplainerFactory
diff --git a/xaib/evaluation/example_selection/case_factory.py b/xaib/evaluation/example_selection/case_factory.py
index a64864c..29c6f37 100644
--- a/xaib/evaluation/example_selection/case_factory.py
+++ b/xaib/evaluation/example_selection/case_factory.py
@@ -1,35 +1,30 @@
-from ...base import Dataset, Model, Factory
+from ...base import Dataset, Factory, Model
 from ...cases.example_selection import (
-    ContinuityCase, ContrastivityCase, CovariateComplexityCase,
-    CorrectnessCase
+    ContinuityCase,
+    ContrastivityCase,
+    CorrectnessCase,
+    CovariateComplexityCase,
 )
-
 from ..utils import NoiseApplier, RandomNeighborsBaseline
 
 
 def continuity(test_ds, model):
     test_ds_noisy = NoiseApplier(test_ds, multiplier=0.01)
-    return ContinuityCase(
-        test_ds,
-        test_ds_noisy,
-        model,
-        multiplier=0.01
-    )
+    return ContinuityCase(test_ds, test_ds_noisy, model, multiplier=0.01)
+
 
 def correctness(test_ds, model):
     noisy_model = RandomNeighborsBaseline(len(test_ds))
 
-    return CorrectnessCase(
-        test_ds,
-        model,
-        noisy_model
-    )
+    return CorrectnessCase(test_ds, model, noisy_model)
 
 
 class CaseFactory(Factory):
     def __init__(self, test_ds: Dataset, model: Model) -> None:
         super().__init__()
-        self._constructors['continuity'] = lambda: continuity(test_ds, model)
-        self._constructors['contrastivity'] = lambda: ContrastivityCase(test_ds, model)
-        self._constructors['covariate_complexity'] = lambda: CovariateComplexityCase(test_ds, model)
-        self._constructors['correctness'] = lambda: correctness(test_ds, model)
+        self._constructors["continuity"] = lambda: continuity(test_ds, model)
+        self._constructors["contrastivity"] = lambda: ContrastivityCase(test_ds, model)
+        self._constructors["covariate_complexity"] = lambda: CovariateComplexityCase(
+            test_ds, model
+        )
+        self._constructors["correctness"] = lambda: correctness(test_ds, model)
diff --git a/xaib/evaluation/example_selection/example_selection.py b/xaib/evaluation/example_selection/example_selection.py
index 645f994..6a0605e 100644
--- a/xaib/evaluation/example_selection/example_selection.py
+++ b/xaib/evaluation/example_selection/example_selection.py
@@ -1,13 +1,11 @@
 import os
 import sys
 
-from cascade.models import ModelRepo
 from cascade import data as cdd
+from cascade.models import ModelRepo
 from cascade.utils.sk_model import SkModel
-
 from xaib.evaluation import DatasetFactory, ModelFactory
-from xaib.evaluation.example_selection import ExplainerFactory, ExperimentFactory
-
+from xaib.evaluation.example_selection import ExperimentFactory, ExplainerFactory
 
 SCRIPT_DIR = os.path.dirname(__file__)
 # xaib/results/...
@@ -16,7 +14,7 @@
 )
 
 sys.path.append(os.path.abspath(os.path.dirname(SCRIPT_DIR)))
-from utils import visualize_results, WrapperModel
+from utils import WrapperModel, visualize_results
 
 
 class SkWrapper(SkModel):
diff --git a/xaib/evaluation/example_selection/experiment_factory.py b/xaib/evaluation/example_selection/experiment_factory.py
index 7508608..6800bdf 100644
--- a/xaib/evaluation/example_selection/experiment_factory.py
+++ b/xaib/evaluation/example_selection/experiment_factory.py
@@ -1,5 +1,6 @@
 from xaib import Factory
 from xaib.evaluation.example_selection import CaseFactory
+
 from ..utils import experiment
 
 
@@ -11,22 +12,21 @@ def __init__(self, repo_path, explainers, test_ds, model, batch_size) -> None:
 
         @experiment(repo_path, explainers=explainers, batch_size=batch_size)
         def continuity():
-            return case_factory.get('continuity')
+            return case_factory.get("continuity")
 
         @experiment(repo_path, explainers=explainers, batch_size=batch_size)
         def contrastivity():
-            return case_factory.get('contrastivity')
+            return case_factory.get("contrastivity")
 
         @experiment(repo_path, explainers=explainers, batch_size=batch_size)
         def covariate_complexity():
-            return case_factory.get('covariate_complexity')
+            return case_factory.get("covariate_complexity")
 
         @experiment(repo_path, explainers=explainers, batch_size=batch_size)
         def correctness():
-            return case_factory.get('correctness')
-
+            return case_factory.get("correctness")
 
-        self._constructors['continuity'] = lambda: continuity
-        self._constructors['contrastivity'] = lambda: contrastivity
-        self._constructors['covariate_complexity'] = lambda: covariate_complexity
-        self._constructors['correctness'] = lambda: correctness
+        self._constructors["continuity"] = lambda: continuity
+        self._constructors["contrastivity"] = lambda: contrastivity
+        self._constructors["covariate_complexity"] = lambda: covariate_complexity
+        self._constructors["correctness"] = lambda: correctness
diff --git a/xaib/evaluation/example_selection/explainer_factory.py b/xaib/evaluation/example_selection/explainer_factory.py
index 5c43818..bac5e92 100644
--- a/xaib/evaluation/example_selection/explainer_factory.py
+++ b/xaib/evaluation/example_selection/explainer_factory.py
@@ -1,12 +1,14 @@
-from ...base import Dataset, Model, Factory
+from ...base import Dataset, Factory, Model
 from ...explainers.example_selection.constant_explainer import ConstantExplainer
-from ...explainers.example_selection.random_explainer import RandomExplainer
 from ...explainers.example_selection.knn_explainer import KNNExplainer
+from ...explainers.example_selection.random_explainer import RandomExplainer
 
 
 class ExplainerFactory(Factory):
     def __init__(self, train_ds: Dataset, model: Model) -> None:
         super().__init__()
-        self._constructors['const'] = lambda: ConstantExplainer(train_ds, train_ds[0]['item'])
-        self._constructors['random'] = lambda: RandomExplainer(train_ds)
-        self._constructors['knn'] = lambda: KNNExplainer(train_ds)        
+        self._constructors["const"] = lambda: ConstantExplainer(
+            train_ds, train_ds[0]["item"]
+        )
+        self._constructors["random"] = lambda: RandomExplainer(train_ds)
+        self._constructors["knn"] = lambda: KNNExplainer(train_ds)
diff --git a/xaib/evaluation/feature_importance/__init__.py b/xaib/evaluation/feature_importance/__init__.py
index 6e56376..520237c 100644
--- a/xaib/evaluation/feature_importance/__init__.py
+++ b/xaib/evaluation/feature_importance/__init__.py
@@ -1,3 +1,3 @@
-from .explainer_factory import ExplainerFactory
 from .case_factory import CaseFactory
 from .experiment_factory import ExperimentFactory
+from .explainer_factory import ExplainerFactory
diff --git a/xaib/evaluation/feature_importance/case_factory.py b/xaib/evaluation/feature_importance/case_factory.py
index 0516deb..637bec1 100644
--- a/xaib/evaluation/feature_importance/case_factory.py
+++ b/xaib/evaluation/feature_importance/case_factory.py
@@ -1,10 +1,12 @@
-from ...base import Dataset, Model, Factory
+from ...base import Dataset, Factory, Model
 from ...cases.feature_importance import (
-    CorrectnessCase, ContinuityCase,
-    ContrastivityCase, CoherenceCase, 
-    CompactnessCase, CovariateComplexityCase
+    CoherenceCase,
+    CompactnessCase,
+    ContinuityCase,
+    ContrastivityCase,
+    CorrectnessCase,
+    CovariateComplexityCase,
 )
-
 from ..utils import NoiseApplier, RandomBinBaseline
 
 
@@ -15,20 +17,17 @@ def correctness(test_ds, model):
 
 def continuity(test_ds, model):
     test_ds_noisy = NoiseApplier(test_ds, multiplier=0.01)
-    return ContinuityCase(
-        test_ds,
-        test_ds_noisy,
-        model,
-        multiplier=0.01
-    )
+    return ContinuityCase(test_ds, test_ds_noisy, model, multiplier=0.01)
 
 
 class CaseFactory(Factory):
     def __init__(self, test_ds: Dataset, model: Model) -> None:
         super().__init__()
-        self._constructors['correctness'] = lambda: correctness(test_ds, model)
-        self._constructors['continuity'] = lambda: continuity(test_ds, model)
-        self._constructors['contrastivity'] = lambda: ContrastivityCase(test_ds, model)
-        self._constructors['coherence'] = lambda: CoherenceCase(test_ds, model)
-        self._constructors['compactness'] = lambda: CompactnessCase(test_ds, model)
-        self._constructors['covariate_complexity'] = lambda: CovariateComplexityCase(test_ds, model)
+        self._constructors["correctness"] = lambda: correctness(test_ds, model)
+        self._constructors["continuity"] = lambda: continuity(test_ds, model)
+        self._constructors["contrastivity"] = lambda: ContrastivityCase(test_ds, model)
+        self._constructors["coherence"] = lambda: CoherenceCase(test_ds, model)
+        self._constructors["compactness"] = lambda: CompactnessCase(test_ds, model)
+        self._constructors["covariate_complexity"] = lambda: CovariateComplexityCase(
+            test_ds, model
+        )
diff --git a/xaib/evaluation/feature_importance/experiment_factory.py b/xaib/evaluation/feature_importance/experiment_factory.py
index 18d47b3..9903066 100644
--- a/xaib/evaluation/feature_importance/experiment_factory.py
+++ b/xaib/evaluation/feature_importance/experiment_factory.py
@@ -1,5 +1,6 @@
 from xaib import Factory
 from xaib.evaluation.feature_importance import CaseFactory
+
 from ..utils import experiment
 
 
@@ -11,36 +12,36 @@ def __init__(self, repo_path, explainers, test_ds, model, batch_size) -> None:
 
         @experiment(repo_path, explainers=explainers, batch_size=batch_size)
         def correctness():
-            return case_factory.get('correctness')
-
+            return case_factory.get("correctness")
 
         @experiment(repo_path, explainers=explainers, batch_size=batch_size)
         def continuity():
-            return case_factory.get('continuity')
-
+            return case_factory.get("continuity")
 
         @experiment(repo_path, explainers=explainers, batch_size=batch_size)
         def contrastivity():
-            return case_factory.get('contrastivity')
-
-
-        @experiment(repo_path, explainers=explainers, expls=list(explainers.values()), batch_size=batch_size)
+            return case_factory.get("contrastivity")
+
+        @experiment(
+            repo_path,
+            explainers=explainers,
+            expls=list(explainers.values()),
+            batch_size=batch_size,
+        )
         def coherence():
-            return case_factory.get('coherence')
-
+            return case_factory.get("coherence")
 
         @experiment(repo_path, explainers=explainers, batch_size=batch_size)
         def compactness():
-            return case_factory.get('compactness')
-
+            return case_factory.get("compactness")
 
         @experiment(repo_path, explainers=explainers, batch_size=batch_size)
         def covariate_complexity():
-            return case_factory.get('covariate_complexity')
-
-        self._constructors['correctness'] = lambda: correctness
-        self._constructors['continuity'] = lambda: continuity
-        self._constructors['contrastivity'] = lambda: contrastivity
-        self._constructors['coherence'] = lambda: coherence
-        self._constructors['compactness'] = lambda: compactness
-        self._constructors['covariate_complexity'] = lambda: covariate_complexity
+            return case_factory.get("covariate_complexity")
+
+        self._constructors["correctness"] = lambda: correctness
+        self._constructors["continuity"] = lambda: continuity
+        self._constructors["contrastivity"] = lambda: contrastivity
+        self._constructors["coherence"] = lambda: coherence
+        self._constructors["compactness"] = lambda: compactness
+        self._constructors["covariate_complexity"] = lambda: covariate_complexity
diff --git a/xaib/evaluation/feature_importance/explainer_factory.py b/xaib/evaluation/feature_importance/explainer_factory.py
index 07cd9de..4466023 100644
--- a/xaib/evaluation/feature_importance/explainer_factory.py
+++ b/xaib/evaluation/feature_importance/explainer_factory.py
@@ -1,14 +1,14 @@
-from ...base import Dataset, Model, Factory
+from ...base import Dataset, Factory, Model
 from ...explainers.feature_importance.constant_explainer import ConstantExplainer
+from ...explainers.feature_importance.lime_explainer import LimeExplainer
 from ...explainers.feature_importance.random_explainer import RandomExplainer
 from ...explainers.feature_importance.shap_explainer import ShapExplainer
-from ...explainers.feature_importance.lime_explainer import LimeExplainer
 
 
 class ExplainerFactory(Factory):
     def __init__(self, train_ds: Dataset, model: Model, labels=[0, 1]) -> None:
         super().__init__()
-        self._constructors['const'] = lambda: ConstantExplainer(constant=1)
-        self._constructors['random'] = lambda: RandomExplainer(shift=-15, magnitude=10)
-        self._constructors['shap'] = lambda: ShapExplainer(train_ds)
-        self._constructors['lime'] = lambda: LimeExplainer(train_ds, labels=labels)
+        self._constructors["const"] = lambda: ConstantExplainer(constant=1)
+        self._constructors["random"] = lambda: RandomExplainer(shift=-15, magnitude=10)
+        self._constructors["shap"] = lambda: ShapExplainer(train_ds)
+        self._constructors["lime"] = lambda: LimeExplainer(train_ds, labels=labels)
diff --git a/xaib/evaluation/model_factory.py b/xaib/evaluation/model_factory.py
index 1070ab1..4e404db 100644
--- a/xaib/evaluation/model_factory.py
+++ b/xaib/evaluation/model_factory.py
@@ -1,8 +1,8 @@
 import numpy as np
 from cascade.utils.sk_model import SkModel
-from sklearn.svm import SVC
-from sklearn.neighbors import KNeighborsClassifier
 from sklearn.metrics import f1_score
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.svm import SVC
 
 from ..base import Factory
 
@@ -14,37 +14,37 @@ def __init__(self, *args, blocks=None, name=None, **kwargs) -> None:
 
 
 def svm(train_ds, test_ds):
-    X_train, Y_train = [x['item'] for x in train_ds], [x['label'] for x in train_ds]
-    X_test, Y_test = [x['item'] for x in test_ds], [x['label'] for x in test_ds]
+    X_train, Y_train = [x["item"] for x in train_ds], [x["label"] for x in train_ds]
+    X_test, Y_test = [x["item"] for x in test_ds], [x["label"] for x in test_ds]
 
     X_train = np.array(X_train)
     Y_train = np.array(Y_train, dtype=int)
     X_test = np.array(X_test)
     Y_test = np.array(Y_test, dtype=int)
 
-    model = SkWrapper(blocks=[SVC(probability=True)], name='svm')
+    model = SkWrapper(blocks=[SVC(probability=True)], name="svm")
     model.fit(X_train, Y_train)
-    model.evaluate(X_test, Y_test, {'f1': f1_score})
+    model.evaluate(X_test, Y_test, {"f1": f1_score})
     return model
 
 
 def knn(train_ds, test_ds):
-    X_train, Y_train = [x['item'] for x in train_ds], [x['label'] for x in train_ds]
-    X_test, Y_test = [x['item'] for x in test_ds], [x['label'] for x in test_ds]
+    X_train, Y_train = [x["item"] for x in train_ds], [x["label"] for x in train_ds]
+    X_test, Y_test = [x["item"] for x in test_ds], [x["label"] for x in test_ds]
 
     X_train = np.array(X_train)
     Y_train = np.array(Y_train, dtype=int)
     X_test = np.array(X_test)
     Y_test = np.array(Y_test, dtype=int)
 
-    model = SkWrapper(blocks=[KNeighborsClassifier(n_neighbors=3)], name='svm')
+    model = SkWrapper(blocks=[KNeighborsClassifier(n_neighbors=3)], name="svm")
     model.fit(X_train, Y_train)
-    model.evaluate(X_test, Y_test, {'f1': f1_score})
+    model.evaluate(X_test, Y_test, {"f1": f1_score})
     return model
 
 
 class ModelFactory(Factory):
     def __init__(self, train_ds, test_ds) -> None:
         super().__init__()
-        self._constructors['svm'] = lambda: svm(train_ds, test_ds)
-        self._constructors['knn'] = lambda: knn(train_ds, test_ds)
+        self._constructors["svm"] = lambda: svm(train_ds, test_ds)
+        self._constructors["knn"] = lambda: knn(train_ds, test_ds)
diff --git a/xaib/evaluation/utils.py b/xaib/evaluation/utils.py
index 13a86c4..38b500c 100644
--- a/xaib/evaluation/utils.py
+++ b/xaib/evaluation/utils.py
@@ -126,11 +126,12 @@ def relative_bar(df):
                 x=df.name.unique(),
                 y=df.loc[df["metric"] == metric]["viz_value"],
                 hovertext=df.loc[df["metric"] == metric]["direction"],
+                width=0.25,
             )
-            for metric in df.metric.unique()
+            for metric in metrics
         ]
     )
-    # Change the bar mode
+
     fig.update_layout(barmode="relative")
 
     return fig
@@ -186,17 +187,33 @@ def visualize_results(path, output_dir=None):
                 pv.loc[i, col] = ""
             prev_val = val
 
-    figs = dict()
-    figs["table"] = table(pv)
+    figs = []
+    figs.append(
+        {
+            "name": "table",
+            "fig": table(pv),
+            "desc": "",
+        }
+    )
+
+    figs.append(
+        {
+            "name": "bar",
+            "fig": relative_bar(df),
+            "desc": "This barchart is simple representation of average scores for each method evaluated.<br>"
+            "Since metrics can be directed differenty e.g. in some cases 'the greater the better' or"
+            " 'the less the better' on this chart all metrics directed down (the less the better) were "
+            "multiplied by -1.<br>This means that on the side below zero better method will be higher, "
+            "closer to zero and on the size above zero better method will be also higher as usual.",
+        }
+    )
 
     metrics = df.metric.unique()
     for name in metrics:
-        figs[f"scatter_{name}"] = scatter(df, name)
-
-    figs["bar"] = relative_bar(df)
+        figs.append({"name": f"scatter_{name}", "fig": scatter(df, name), "desc": ""})
 
-    for name in figs:
+    for fig in figs:
         if output_dir is not None:
-            figs[name].write_image(os.path.join(output_dir, name + ".png"))
+            fig["fig"].write_image(os.path.join(output_dir, fig["name"] + ".png"))
 
     return figs
diff --git a/xaib/explainers/example_selection/constant_explainer.py b/xaib/explainers/example_selection/constant_explainer.py
index caa7d3d..6a97408 100644
--- a/xaib/explainers/example_selection/constant_explainer.py
+++ b/xaib/explainers/example_selection/constant_explainer.py
@@ -1,4 +1,5 @@
 import numpy as np
+
 from ...base import Explainer
 
 
diff --git a/xaib/explainers/example_selection/knn_explainer.py b/xaib/explainers/example_selection/knn_explainer.py
index 32cc8fc..8f1064b 100644
--- a/xaib/explainers/example_selection/knn_explainer.py
+++ b/xaib/explainers/example_selection/knn_explainer.py
@@ -1,6 +1,7 @@
-from ...base import Explainer
 import numpy as np
 
+from ...base import Explainer
+
 
 class KNNExplainer(Explainer):
     def __init__(self, train_ds, **kwargs) -> None:
@@ -9,4 +10,4 @@ def __init__(self, train_ds, **kwargs) -> None:
 
     def predict(self, x, model):
         _, indices = model._pipeline[0].kneighbors(x, 1)
-        return np.asarray([self._train_ds[i[0]]['item'] for i in indices])
+        return np.asarray([self._train_ds[i[0]]["item"] for i in indices])
diff --git a/xaib/explainers/example_selection/random_explainer.py b/xaib/explainers/example_selection/random_explainer.py
index c4c645b..95c141f 100644
--- a/xaib/explainers/example_selection/random_explainer.py
+++ b/xaib/explainers/example_selection/random_explainer.py
@@ -9,5 +9,9 @@ def __init__(self, train_ds, **kwargs) -> None:
         self._train_ds = train_ds
 
     def predict(self, x, model):
-        return np.asarray([self._train_ds[np.random.randint(0, len(self._train_ds) - 1)]['item']
-            for _ in range(len(x))])
+        return np.asarray(
+            [
+                self._train_ds[np.random.randint(0, len(self._train_ds) - 1)]["item"]
+                for _ in range(len(x))
+            ]
+        )
diff --git a/xaib/explainers/feature_importance/constant_explainer.py b/xaib/explainers/feature_importance/constant_explainer.py
index f679f12..0e7b67f 100644
--- a/xaib/explainers/feature_importance/constant_explainer.py
+++ b/xaib/explainers/feature_importance/constant_explainer.py
@@ -1,11 +1,12 @@
-from typing import Any, Union, Dict
+from typing import Any, Dict, Union
+
 import numpy as np
 
-from ...base import Model, Explainer
+from ...base import Explainer, Model
 
 
 class ConstantExplainer(Explainer):
-    def __init__(self, constant: Any, *args:Any, **kwargs: Any) -> None:
+    def __init__(self, constant: Any, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.constant = constant
 
diff --git a/xaib/explainers/feature_importance/lime_explainer.py b/xaib/explainers/feature_importance/lime_explainer.py
index 9f08ea1..82af208 100644
--- a/xaib/explainers/feature_importance/lime_explainer.py
+++ b/xaib/explainers/feature_importance/lime_explainer.py
@@ -1,5 +1,5 @@
-from lime import lime_tabular
 import numpy as np
+from lime import lime_tabular
 
 from ...base import Explainer
 
@@ -8,17 +8,21 @@ class LimeExplainer(Explainer):
     def __init__(self, train_ds, labels, *args, meta_prefix=None, **kwargs) -> None:
         super().__init__(*args, meta_prefix=meta_prefix, **kwargs)
 
-        data = np.array([item['item'] for item in train_ds])
-        self._explainer = lime_tabular.LimeTabularExplainer(data, feature_selection='none')
+        data = np.array([item["item"] for item in train_ds])
+        self._explainer = lime_tabular.LimeTabularExplainer(
+            data, feature_selection="none"
+        )
         self._labels = labels
 
     def predict(self, x, model):
-        if not hasattr(model, 'predict_proba'):
-            raise ValueError('The model should have `predict_proba` method')
+        if not hasattr(model, "predict_proba"):
+            raise ValueError("The model should have `predict_proba` method")
 
         explanations = []
         for item in x:
-            ex = self._explainer.explain_instance(item, model.predict_proba, labels=self._labels)
+            ex = self._explainer.explain_instance(
+                item, model.predict_proba, labels=self._labels
+            )
             predicted_label = np.argmax(ex.predict_proba)
             ex = ex.as_map()[predicted_label]
             # Sorts by the feature order
diff --git a/xaib/explainers/feature_importance/random_explainer.py b/xaib/explainers/feature_importance/random_explainer.py
index 9cfabc5..98dc274 100644
--- a/xaib/explainers/feature_importance/random_explainer.py
+++ b/xaib/explainers/feature_importance/random_explainer.py
@@ -1,6 +1,8 @@
 from typing import Any
+
 import numpy as np
-from ...base import Model, Explainer
+
+from ...base import Explainer, Model
 
 
 class RandomExplainer(Explainer):
diff --git a/xaib/explainers/feature_importance/shap_explainer.py b/xaib/explainers/feature_importance/shap_explainer.py
index 87efef9..d494d61 100644
--- a/xaib/explainers/feature_importance/shap_explainer.py
+++ b/xaib/explainers/feature_importance/shap_explainer.py
@@ -8,7 +8,7 @@ class ShapExplainer(Explainer):
     def __init__(self, train_ds, *args, meta_prefix=None, **kwargs) -> None:
         super().__init__(*args, meta_prefix=meta_prefix, **kwargs)
         self._train_ds = train_ds
-        self._train_data = np.array([item['item'] for item in self._train_ds])
+        self._train_data = np.array([item["item"] for item in self._train_ds])
 
     def predict(self, x, model):
         explainer = shap.Explainer(model.predict, self._train_data)
diff --git a/xaib/metrics/example_selection/__init__.py b/xaib/metrics/example_selection/__init__.py
index 6604f18..9f6af71 100644
--- a/xaib/metrics/example_selection/__init__.py
+++ b/xaib/metrics/example_selection/__init__.py
@@ -1,4 +1,4 @@
-from .small_noise_check import SmallNoiseCheck
-from .target_discriminativeness import TargetDiscriminativeness
 from .covariate_regularity import CovariateRegularity
 from .parameter_randomization_check import ParameterRandomizationCheck
+from .small_noise_check import SmallNoiseCheck
+from .target_discriminativeness import TargetDiscriminativeness
diff --git a/xaib/metrics/example_selection/covariate_regularity.py b/xaib/metrics/example_selection/covariate_regularity.py
index 4e56bc7..3ef4a04 100644
--- a/xaib/metrics/example_selection/covariate_regularity.py
+++ b/xaib/metrics/example_selection/covariate_regularity.py
@@ -1,10 +1,10 @@
-from typing import Dict, List, Any, Union
+from typing import Any, Dict, Union
 
 import numpy as np
 from tqdm import tqdm
 
-from ...base import Dataset, Model, Metric, Explainer
-from ...utils import entropy, minmax_normalize, SimpleDataloader, Filter
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import Filter, SimpleDataloader, entropy, minmax_normalize
 
 
 class CovariateRegularity(Metric):
@@ -13,23 +13,28 @@ class CovariateRegularity(Metric):
     complies with domain knowledge, ground-truth
     or other methods
     """
+
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'covariate_regularity'
-        self.direction = 'down'
+        self.name = "covariate_regularity"
+        self.direction = "down"
 
     def compute(
         self,
         expl: Explainer,
         batch_size: int = 1,
-        expl_kwargs: Union[Dict[Any, Any], None] = None
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
     ) -> None:
         if expl_kwargs is None:
-            expl_kwargs = {} 
+            expl_kwargs = {}
 
         # Obtain all the labels
-        labels = np.asarray([item['label'] for item in
-                             tqdm(self._ds, desc='Obtaining labels', leave=False)])
+        labels = np.asarray(
+            [
+                item["label"]
+                for item in tqdm(self._ds, desc="Obtaining labels", leave=False)
+            ]
+        )
 
         # Determine how much of an intersection different labels have
         unique_labels, counts = np.unique(labels, return_counts=True)
@@ -43,7 +48,7 @@ def compute(
         for u in unique_labels:
             dl = SimpleDataloader(Filter(self._ds, coords[u]), batch_size=batch_size)
             for batch in tqdm(dl):
-                ex = expl.predict(batch['item'], self._model, **expl_kwargs)
+                ex = expl.predict(batch["item"], self._model, **expl_kwargs)
                 ex = minmax_normalize(ex)
 
                 explanations[u] += ex.tolist()
diff --git a/xaib/metrics/example_selection/parameter_randomization_check.py b/xaib/metrics/example_selection/parameter_randomization_check.py
index 6192826..f2ce7dd 100644
--- a/xaib/metrics/example_selection/parameter_randomization_check.py
+++ b/xaib/metrics/example_selection/parameter_randomization_check.py
@@ -1,23 +1,27 @@
-from typing import Any, Union, Dict
+from typing import Any, Dict, Union
+
 import numpy as np
 from tqdm import tqdm
-from ...base import Metric, Explainer, Dataset, Model
-from ...utils import batch_count_eq, minmax_normalize, SimpleDataloader
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_count_eq, minmax_normalize
 
 
 class ParameterRandomizationCheck(Metric):
-    def __init__(self, ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any) -> None:
+    def __init__(
+        self, ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, **kwargs)
         self._noisy_model = noisy_model
-        self.name = 'parameter_randomization_check'
-        self.direction = 'down'
+        self.name = "parameter_randomization_check"
+        self.direction = "down"
 
     def compute(
         self,
         expl: Explainer,
         batch_size: int = 1,
         expl_kwargs: Union[Dict[Any, Any], None] = None,
-        expl_noisy_kwargs: Union[Dict[Any, Any], None] = None
+        expl_noisy_kwargs: Union[Dict[Any, Any], None] = None,
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
@@ -27,10 +31,12 @@ def compute(
         diffs_expl = []
 
         for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
-            item = batch['item']
+            item = batch["item"]
 
             explanation_batch = expl.predict(item, self._model, **expl_kwargs)
-            noisy_explanation_batch = expl.predict(item, self._noisy_model, **expl_noisy_kwargs)
+            noisy_explanation_batch = expl.predict(
+                item, self._noisy_model, **expl_noisy_kwargs
+            )
 
             diffs_expl += batch_count_eq(explanation_batch, noisy_explanation_batch)
 
diff --git a/xaib/metrics/example_selection/small_noise_check.py b/xaib/metrics/example_selection/small_noise_check.py
index fcd8cb2..22180fe 100644
--- a/xaib/metrics/example_selection/small_noise_check.py
+++ b/xaib/metrics/example_selection/small_noise_check.py
@@ -1,10 +1,10 @@
-from typing import Dict, Any, Union
+from typing import Any, Dict, Union
 
 import numpy as np
 from tqdm import tqdm
 
-from ...base import Metric, Explainer, Model, Dataset
-from ...utils import batch_count_eq, SimpleDataloader
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_count_eq
 
 
 class SmallNoiseCheck(Metric):
@@ -14,23 +14,18 @@ class SmallNoiseCheck(Metric):
     """
 
     def __init__(
-        self,
-        ds: Dataset,
-        noisy_ds: Dataset,
-        model: Model,
-        *args: Any,
-        **kwargs: Any
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
     ) -> None:
         super().__init__(ds, model, *args, **kwargs)
         self._noisy_ds = noisy_ds
-        self.name = 'small_noise_check'
-        self.direction = 'up'
+        self.name = "small_noise_check"
+        self.direction = "up"
 
     def compute(
         self,
         expl: Explainer,
         batch_size: int = 1,
-        expl_kwargs: Union[Dict[Any, Any], None] = None
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
@@ -38,14 +33,16 @@ def compute(
         counts = []
         dls = zip(
             SimpleDataloader(self._ds, batch_size),
-            SimpleDataloader(self._noisy_ds, batch_size)
+            SimpleDataloader(self._noisy_ds, batch_size),
         )
         for batch, noisy_batch in tqdm(dls):
-            item = batch['item']
-            noisy_item = noisy_batch['item']
+            item = batch["item"]
+            noisy_item = noisy_batch["item"]
 
             explanation_batch = expl.predict(item, self._model, **expl_kwargs)
-            noisy_explanation_batch = expl.predict(noisy_item, self._model, **expl_kwargs)
+            noisy_explanation_batch = expl.predict(
+                noisy_item, self._model, **expl_kwargs
+            )
 
             counts += batch_count_eq(explanation_batch, noisy_explanation_batch)
 
diff --git a/xaib/metrics/example_selection/target_discriminativeness.py b/xaib/metrics/example_selection/target_discriminativeness.py
index 18f9b51..466c17e 100644
--- a/xaib/metrics/example_selection/target_discriminativeness.py
+++ b/xaib/metrics/example_selection/target_discriminativeness.py
@@ -1,15 +1,16 @@
-from tqdm import tqdm
 from cascade import data as cdd
+from tqdm import tqdm
+
 from ...base import Metric
-from ...utils import SimpleDataloader, KeyedComposer
 from ...evaluation import ModelFactory
+from ...utils import KeyedComposer, SimpleDataloader
 
 
 class TargetDiscriminativeness(Metric):
     def __init__(self, ds, model, *args, **kwargs):
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'target_discriminativeness'
-        self.direction = 'up'
+        self.name = "target_discriminativeness"
+        self.direction = "up"
 
     def compute(self, explainer, batch_size=1, expl_kwargs=None):
         if expl_kwargs is None:
@@ -18,7 +19,7 @@ def compute(self, explainer, batch_size=1, expl_kwargs=None):
         labels = []
         explanations = []
         for batch in tqdm(SimpleDataloader(self._ds, batch_size=batch_size)):
-            item, label = batch['item'], batch['label']
+            item, label = batch["item"], batch["label"]
 
             explanation_batch = explainer.predict(item, self._model, **expl_kwargs)
 
@@ -26,14 +27,11 @@ def compute(self, explainer, batch_size=1, expl_kwargs=None):
             labels += label.tolist()
 
         ds = KeyedComposer(
-            {
-                'item': cdd.Wrapper(explanations),
-                'label': cdd.Wrapper(labels)
-            }
+            {"item": cdd.Wrapper(explanations), "label": cdd.Wrapper(labels)}
         )
 
         expl_train_ds, expl_test_ds = cdd.split(ds, frac=0.8)
 
-        user_model = ModelFactory(expl_train_ds, expl_test_ds).get('svm')
+        user_model = ModelFactory(expl_train_ds, expl_test_ds).get("svm")
 
-        return user_model.metrics['f1']
+        return user_model.metrics["f1"]
diff --git a/xaib/metrics/feature_importance/__init__.py b/xaib/metrics/feature_importance/__init__.py
index 2dcaa80..96b497c 100644
--- a/xaib/metrics/feature_importance/__init__.py
+++ b/xaib/metrics/feature_importance/__init__.py
@@ -1,6 +1,6 @@
-from .small_noise_check import SmallNoiseCheck
+from .covariate_regularity import CovariateRegularity
 from .label_difference import LabelDifference
-from .parameter_randomization_check import ParameterRandomizationCheck
 from .other_disagreement import OtherDisagreement
+from .parameter_randomization_check import ParameterRandomizationCheck
+from .small_noise_check import SmallNoiseCheck
 from .sparsity import Sparsity
-from .covariate_regularity import CovariateRegularity
diff --git a/xaib/metrics/feature_importance/covariate_regularity.py b/xaib/metrics/feature_importance/covariate_regularity.py
index 1f4984b..6617eab 100644
--- a/xaib/metrics/feature_importance/covariate_regularity.py
+++ b/xaib/metrics/feature_importance/covariate_regularity.py
@@ -1,9 +1,10 @@
-from typing import Dict, Union, Any
+from typing import Any, Dict, Union
+
 import numpy as np
 from tqdm import tqdm
 
-from ...base import Dataset, Model, Metric, Explainer
-from ...utils import entropy, minmax_normalize, SimpleDataloader
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, entropy, minmax_normalize
 
 
 class CovariateRegularity(Metric):
@@ -12,23 +13,24 @@ class CovariateRegularity(Metric):
     consistency. If explanation features are noisy, then
     they are harder to remember.
     """
+
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'covariate_regularity'
-        self.direction = 'down'
+        self.name = "covariate_regularity"
+        self.direction = "down"
 
     def compute(
-            self,
-            expl: Explainer,
-            batch_size: int = 1,
-            expl_kwargs: Union[Dict[Any, Any], None] = None,
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
     ):
         if expl_kwargs is None:
             expl_kwargs = {}
 
         explanations = []
         for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
-            item = batch['item']
+            item = batch["item"]
 
             e = expl.predict(item, self._model, **expl_kwargs)
             e = minmax_normalize(e)
diff --git a/xaib/metrics/feature_importance/label_difference.py b/xaib/metrics/feature_importance/label_difference.py
index 31a8abe..0d62054 100644
--- a/xaib/metrics/feature_importance/label_difference.py
+++ b/xaib/metrics/feature_importance/label_difference.py
@@ -1,11 +1,11 @@
-from typing import List, Union, Dict, Any
+from typing import Any, Dict, Union
 
-from tqdm import tqdm
 import numpy as np
 from cascade.data import Sampler
+from tqdm import tqdm
 
-from ...utils import batch_rmse, minmax_normalize, SimpleDataloader, Filter
-from ...base import Dataset, Model, Metric, Explainer
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import Filter, SimpleDataloader, batch_rmse, minmax_normalize
 
 
 class LabelDifference(Metric):
@@ -13,23 +13,28 @@ class LabelDifference(Metric):
     ContrastivityCase Measures how different explanations
     are actually different from each other
     """
+
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'label_difference'
-        self.direction = 'up'
+        self.name = "label_difference"
+        self.direction = "up"
 
     def compute(
         self,
         expl: Explainer,
         batch_size: int = 1,
-        expl_kwargs: Union[Dict[Any, Any], None] = None
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
     ) -> None:
         if expl_kwargs is None:
-            expl_kwargs = {} 
+            expl_kwargs = {}
 
         # Obtain all the labels
-        labels = np.asarray([item['label'] for item in
-                             tqdm(self._ds, desc='Obtaining labels', leave=False)])
+        labels = np.asarray(
+            [
+                item["label"]
+                for item in tqdm(self._ds, desc="Obtaining labels", leave=False)
+            ]
+        )
 
         # Determine how much of an intersection different labels have
         unique_labels, counts = np.unique(labels, return_counts=True)
@@ -43,7 +48,7 @@ def compute(
         for u in unique_labels:
             dl = SimpleDataloader(Filter(self._ds, coords[u]), batch_size=batch_size)
             for batch in tqdm(dl):
-                ex = expl.predict(batch['item'], self._model, **expl_kwargs)
+                ex = expl.predict(batch["item"], self._model, **expl_kwargs)
                 ex = minmax_normalize(ex)
                 explanations[u].append(ex)
 
@@ -55,8 +60,7 @@ def compute(
                     continue
 
                 for batch, other_batch in zip(
-                    explanations[u],
-                    explanations[other_label]
+                    explanations[u], explanations[other_label]
                 ):
                     rmse = batch_rmse(batch, other_batch)
                     diffs[u] += list(rmse)
diff --git a/xaib/metrics/feature_importance/other_disagreement.py b/xaib/metrics/feature_importance/other_disagreement.py
index 0e1f88c..86db5cf 100644
--- a/xaib/metrics/feature_importance/other_disagreement.py
+++ b/xaib/metrics/feature_importance/other_disagreement.py
@@ -1,10 +1,10 @@
-from typing import Dict, List, Any, Union
+from typing import Any, Dict, List, Union
 
 import numpy as np
 from tqdm import tqdm
 
-from ...base import Dataset, Model, Metric, Explainer
-from ...utils import batch_rmse, minmax_normalize, SimpleDataloader
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_rmse, minmax_normalize
 
 
 class OtherDisagreement(Metric):
@@ -13,10 +13,11 @@ class OtherDisagreement(Metric):
     complies with domain knowledge, ground-truth
     or other methods
     """
+
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'other_disagreement'
-        self.direction = 'down'
+        self.name = "other_disagreement"
+        self.direction = "down"
 
     def compute(
         self,
@@ -24,7 +25,7 @@ def compute(
         batch_size: int = 1,
         expls: Union[List[Explainer], None] = None,
         expl_kwargs: Union[Dict[Any, Any], None] = None,
-        expls_kwargs: Union[List[Dict[Any, Any]], None] = None
+        expls_kwargs: Union[List[Dict[Any, Any]], None] = None,
     ) -> None:
         # Default initialization in case of empty
         # parameter to not to break default argument order
@@ -36,15 +37,16 @@ def compute(
         if expls_kwargs is None:
             expls_kwargs = [{} for _ in range(len(expls))]
 
-
         diffs = []
         for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
-            item = batch['item']
+            item = batch["item"]
 
             e = expl.predict(item, self._model, **expl_kwargs)
             e = minmax_normalize(e)
-            other_e = [minmax_normalize(other_expl.predict(item, self._model, **other_kwargs))
-                       for other_expl, other_kwargs in zip(expls, expls_kwargs)]
+            other_e = [
+                minmax_normalize(other_expl.predict(item, self._model, **other_kwargs))
+                for other_expl, other_kwargs in zip(expls, expls_kwargs)
+            ]
 
             for oe in other_e:
                 diffs += batch_rmse(e, oe)
diff --git a/xaib/metrics/feature_importance/parameter_randomization_check.py b/xaib/metrics/feature_importance/parameter_randomization_check.py
index 5440e13..5a0cf82 100644
--- a/xaib/metrics/feature_importance/parameter_randomization_check.py
+++ b/xaib/metrics/feature_importance/parameter_randomization_check.py
@@ -1,8 +1,10 @@
-from typing import Any, Union, Dict
+from typing import Any, Dict, Union
+
 import numpy as np
 from tqdm import tqdm
-from ...base import Metric, Explainer, Dataset, Model
-from ...utils import batch_rmse, minmax_normalize, SimpleDataloader
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_rmse, minmax_normalize
 
 
 class ParameterRandomizationCheck(Metric):
diff --git a/xaib/metrics/feature_importance/small_noise_check.py b/xaib/metrics/feature_importance/small_noise_check.py
index 2bdf094..381f0d6 100644
--- a/xaib/metrics/feature_importance/small_noise_check.py
+++ b/xaib/metrics/feature_importance/small_noise_check.py
@@ -1,10 +1,10 @@
-from typing import Dict, Any, Union
+from typing import Any, Dict, Union
 
 import numpy as np
 from tqdm import tqdm
 
-from ...base import Metric, Explainer, Model, Dataset
-from ...utils import batch_rmse, minmax_normalize, SimpleDataloader
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_rmse, minmax_normalize
 
 
 class SmallNoiseCheck(Metric):
@@ -12,24 +12,20 @@ class SmallNoiseCheck(Metric):
     Continuity measures how continuous
     explanation function is
     """
+
     def __init__(
-        self,
-        ds: Dataset,
-        noisy_ds: Dataset,
-        model: Model,
-        *args: Any,
-        **kwargs: Any
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
     ) -> None:
         super().__init__(ds, model, *args, **kwargs)
         self._noisy_ds = noisy_ds
-        self.name = 'small_noise_check'
-        self.direction = 'down'
+        self.name = "small_noise_check"
+        self.direction = "down"
 
     def compute(
         self,
         expl: Explainer,
         batch_size: int = 1,
-        expl_kwargs: Union[Dict[Any, Any], None] = None
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
@@ -37,14 +33,16 @@ def compute(
         rmses = []
         dls = zip(
             SimpleDataloader(self._ds, batch_size),
-            SimpleDataloader(self._noisy_ds, batch_size)
+            SimpleDataloader(self._noisy_ds, batch_size),
         )
         for batch, noisy_batch in tqdm(dls):
-            item = batch['item']
-            noisy_item = noisy_batch['item']
+            item = batch["item"]
+            noisy_item = noisy_batch["item"]
 
             explanation_batch = expl.predict(item, self._model, **expl_kwargs)
-            noisy_explanation_batch = expl.predict(noisy_item, self._model, **expl_kwargs)
+            noisy_explanation_batch = expl.predict(
+                noisy_item, self._model, **expl_kwargs
+            )
 
             explanation_batch = minmax_normalize(explanation_batch)
             noisy_explanation_batch = minmax_normalize(noisy_explanation_batch)
diff --git a/xaib/metrics/feature_importance/sparsity.py b/xaib/metrics/feature_importance/sparsity.py
index 88105e0..2d6ce70 100644
--- a/xaib/metrics/feature_importance/sparsity.py
+++ b/xaib/metrics/feature_importance/sparsity.py
@@ -1,10 +1,10 @@
-from typing import Dict, Union, Any
+from typing import Any, Dict, Union
 
 import numpy as np
 from tqdm import tqdm
 
-from ...base import Dataset, Model, Metric, Explainer
-from ...utils import batch_gini, minmax_normalize, SimpleDataloader
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_gini, minmax_normalize
 
 
 class Sparsity(Metric):
@@ -13,16 +13,17 @@ class Sparsity(Metric):
     representations are. Explanations are more
     understandable if they are short.
     """
+
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'sparsity'
-        self.direction = 'up'
+        self.name = "sparsity"
+        self.direction = "up"
 
     def compute(
         self,
         expl: Explainer,
         batch_size: int = 1,
-        expl_kwargs: Union[Dict[Any, Any], None] = None
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
@@ -30,7 +31,7 @@ def compute(
         ginis = []
 
         for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
-            item = batch['item']
+            item = batch["item"]
 
             explanation_batch = expl.predict(item, self._model, **expl_kwargs)
             explanation_batch = minmax_normalize(explanation_batch)
diff --git a/xaib/tests/test_feature_importance_shap.py b/xaib/tests/test_feature_importance_shap.py
deleted file mode 100644
index cff9d28..0000000
--- a/xaib/tests/test_feature_importance_shap.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import os
-import sys
-
-from .conftest import test_feature_importance_explainer_interface
-
-PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.append(PROJECT_DIR)
-from xaib.explainers.shap_explainer import ShapExplainer
-
-
-def test_explainer(train_test_pair, dummy_model):
-    train_x, test_x, train_y, test_y = train_test_pair
-    explainer = ShapExplainer(train_x)
-
-    test_feature_importance_explainer_interface(explainer, dummy_model, test_x)
diff --git a/xaib/utils/__init__.py b/xaib/utils/__init__.py
index 378dd1a..304cf20 100644
--- a/xaib/utils/__init__.py
+++ b/xaib/utils/__init__.py
@@ -1,4 +1,10 @@
 from .utils import (
-    batch_rmse, minmax_normalize, entropy, batch_gini, SimpleDataloader,
-    batch_count_eq, KeyedComposer, Filter
+    Filter,
+    KeyedComposer,
+    SimpleDataloader,
+    batch_count_eq,
+    batch_gini,
+    batch_rmse,
+    entropy,
+    minmax_normalize,
 )
diff --git a/xaib/utils/utils.py b/xaib/utils/utils.py
index 586df9b..80221d0 100644
--- a/xaib/utils/utils.py
+++ b/xaib/utils/utils.py
@@ -1,6 +1,7 @@
 from typing import Dict, List
+
 import numpy as np
-from cascade.data import SizedDataset, Composer, Sampler
+from cascade.data import Composer, Sampler, SizedDataset
 
 
 def rmse(x, y):
@@ -70,7 +71,7 @@ def __getitem__(self, index: int) -> Dict:
             item = self._data[i]
             for ch in self._channels:
                 batch[ch].append(item[ch])
-        
+
         for ch in batch:
             batch[ch] = np.array(batch[ch])
 

From 58ab39034a67787aa338eaac85b44a45adb2b834 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sun, 2 Apr 2023 12:58:55 +0300
Subject: [PATCH 13/19] Fill in metrics docs

---
 xaib/docs/source/modules.rst                  |  3 +
 .../source/xaib/metrics.example_selection.rst | 38 +++++++++++++
 .../xaib/metrics.feature_importance.rst       | 56 +++++++++++++++++++
 xaib/docs/source/xaib/metrics.rst             |  6 ++
 .../example_selection/covariate_regularity.py | 13 ++++-
 .../parameter_randomization_check.py          | 23 ++++++++
 .../example_selection/small_noise_check.py    | 16 +++++-
 .../target_discriminativeness.py              | 12 ++++
 .../covariate_regularity.py                   | 13 ++++-
 .../feature_importance/label_difference.py    | 13 ++++-
 .../feature_importance/other_disagreement.py  |  7 ++-
 .../parameter_randomization_check.py          | 19 ++++++-
 .../feature_importance/small_noise_check.py   |  8 ++-
 xaib/metrics/feature_importance/sparsity.py   | 12 +++-
 14 files changed, 218 insertions(+), 21 deletions(-)
 create mode 100644 xaib/docs/source/xaib/metrics.example_selection.rst
 create mode 100644 xaib/docs/source/xaib/metrics.feature_importance.rst
 create mode 100644 xaib/docs/source/xaib/metrics.rst

diff --git a/xaib/docs/source/modules.rst b/xaib/docs/source/modules.rst
index 9d4ee42..601de40 100644
--- a/xaib/docs/source/modules.rst
+++ b/xaib/docs/source/modules.rst
@@ -4,5 +4,8 @@ XAIB modules reference
 .. toctree::
     xaib/base
     xaib/cases
+    xaib/datasets
     xaib/explainers
+    xaib/metrics
+    xaib/models
     xaib/utils
diff --git a/xaib/docs/source/xaib/metrics.example_selection.rst b/xaib/docs/source/xaib/metrics.example_selection.rst
new file mode 100644
index 0000000..c92a891
--- /dev/null
+++ b/xaib/docs/source/xaib/metrics.example_selection.rst
@@ -0,0 +1,38 @@
+xaib.metrics.example_selection
+==============================
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.CovariateRegularity
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.ParameterRandomizationCheck
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.SmallNoiseCheck
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.TargetDiscriminativeness
+   :members:
+
+|
+
+|
+
+|
diff --git a/xaib/docs/source/xaib/metrics.feature_importance.rst b/xaib/docs/source/xaib/metrics.feature_importance.rst
new file mode 100644
index 0000000..cb2f047
--- /dev/null
+++ b/xaib/docs/source/xaib/metrics.feature_importance.rst
@@ -0,0 +1,56 @@
+xaib.metrics.feature_importance
+===============================
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.CovariateRegularity
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.LabelDifference
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.OtherDisagreement
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.ParameterRandomizationCheck
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.SmallNoiseCheck
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.Sparsity
+   :members:
+
+|
+
+|
+
+|
\ No newline at end of file
diff --git a/xaib/docs/source/xaib/metrics.rst b/xaib/docs/source/xaib/metrics.rst
new file mode 100644
index 0000000..e8fc312
--- /dev/null
+++ b/xaib/docs/source/xaib/metrics.rst
@@ -0,0 +1,6 @@
+xaib.metrics
+============
+
+.. toctree::
+    metrics.feature_importance
+    metrics.example_selection
diff --git a/xaib/metrics/example_selection/covariate_regularity.py b/xaib/metrics/example_selection/covariate_regularity.py
index 3ef4a04..e127e62 100644
--- a/xaib/metrics/example_selection/covariate_regularity.py
+++ b/xaib/metrics/example_selection/covariate_regularity.py
@@ -9,9 +9,16 @@
 
 class CovariateRegularity(Metric):
     """
-    Coherence measures how method
-    complies with domain knowledge, ground-truth
-    or other methods
+    Measures how noisy the features in the examples provided
+    by the method.
+
+    More simple explanations are considered better.
+    This is measured by average Shannon entropy over batch-normalized explanations.
+
+    **The less the better**
+
+      - **Worst case:** constant explainer that gives examples with same importance to each feature, that is equal to 1/N where N is the number of features
+      - **Best case:** constant explainer that gives examples with one feature with maximum value and others zero
     """
 
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
diff --git a/xaib/metrics/example_selection/parameter_randomization_check.py b/xaib/metrics/example_selection/parameter_randomization_check.py
index f2ce7dd..7c6038d 100644
--- a/xaib/metrics/example_selection/parameter_randomization_check.py
+++ b/xaib/metrics/example_selection/parameter_randomization_check.py
@@ -8,6 +8,29 @@
 
 
 class ParameterRandomizationCheck(Metric):
+    """
+    Parameter randomization check is a sanity-check.
+    To ensure that the model influence explanations the
+    following is done. The model is changed and it is expected that
+    explanations should not stay the same is model changed.
+    This check uses random model baselines instead of same models
+    with randomized internal states.
+    Then the explanations on the original data are obtained.
+    They are compared with explanations done with the original model by
+    counting how many examples were the same for same data points.
+    The less  the better.
+
+    **The less the better**
+     - **Worst case:** explanations are the same, so it is Constant explainer
+     - **Best case:** is reached when explanations are the opposite,
+     distance between them maximized.
+    The problem with this kind of metric is
+    with its maximization. It seems redundant to maximize it because more
+    different explanations on random states do not mean that the model is
+    more correct.
+    It is difficult to define best case explainer in this case - the metric has no maximum value.
+    """
+
     def __init__(
         self, ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any
     ) -> None:
diff --git a/xaib/metrics/example_selection/small_noise_check.py b/xaib/metrics/example_selection/small_noise_check.py
index 22180fe..66e5ebb 100644
--- a/xaib/metrics/example_selection/small_noise_check.py
+++ b/xaib/metrics/example_selection/small_noise_check.py
@@ -9,8 +9,20 @@
 
 class SmallNoiseCheck(Metric):
     """
-    Continuity measures how continuous
-    explanation function is
+    Since the nature of example selection methods is discrete - we choose an example
+    from finite set, the use of RMSE measure may be not appropriate.
+    This means that the metric that is similar to the one that was used in feature
+    importance case is not suitable.
+    More appropriate metric could be the following: take the test dataset and add
+    small amount of noise to the items as was done in the feature importance case.
+    Then count the number of item pairs when the example provided didn't change and
+    divide by the total number of items.
+    This ratio then is how continuous example generator is - if it provides the same
+    examples for slightly changed inputs, then it is continuous.
+
+    **The less the better**
+     - **Worst case:** Constant explainer
+     - **Best case:** Random explainer
     """
 
     def __init__(
diff --git a/xaib/metrics/example_selection/target_discriminativeness.py b/xaib/metrics/example_selection/target_discriminativeness.py
index 466c17e..b179bc2 100644
--- a/xaib/metrics/example_selection/target_discriminativeness.py
+++ b/xaib/metrics/example_selection/target_discriminativeness.py
@@ -7,6 +7,18 @@
 
 
 class TargetDiscriminativeness(Metric):
+    """
+    Given true labels and explanations in form of the examples, train another model to discriminate between labels.
+    The quality of the model on the examples given describes the quality of the explanations.
+    The quality can be measured by any performance metric,
+    but it is better to adopt to imbalanced data and use F1-measure for example.
+
+    **The greater the better**
+
+    **Best case:** examples are descriptive of labels so that the model reaches best performance
+    **Worst case:** constant or random baseline - giving insufficient information to grasp labels
+    """
+
     def __init__(self, ds, model, *args, **kwargs):
         super().__init__(ds, model, *args, **kwargs)
         self.name = "target_discriminativeness"
diff --git a/xaib/metrics/feature_importance/covariate_regularity.py b/xaib/metrics/feature_importance/covariate_regularity.py
index 6617eab..5a2dbb1 100644
--- a/xaib/metrics/feature_importance/covariate_regularity.py
+++ b/xaib/metrics/feature_importance/covariate_regularity.py
@@ -9,9 +9,16 @@
 
 class CovariateRegularity(Metric):
     """
-    CovariateComplexity Measures how complex explanation features are, their
-    consistency. If explanation features are noisy, then
-    they are harder to remember.
+    Covariate Regularity using entropy over explanations
+
+    This measures how comprehensible the explanations are in average.
+    More simple explanations are considered better.
+    This is measured by average Shannon entropy over batch-normalized explanations.
+
+    **The less the better**
+
+      - **Worst case:** constant explainer that gives same importance to each feature, that is equal to 1/N where N is the number of features
+      - **Best case:** constant explainer that gives one feature maximum value and others zero
     """
 
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
diff --git a/xaib/metrics/feature_importance/label_difference.py b/xaib/metrics/feature_importance/label_difference.py
index 0d62054..9167bb5 100644
--- a/xaib/metrics/feature_importance/label_difference.py
+++ b/xaib/metrics/feature_importance/label_difference.py
@@ -10,8 +10,17 @@
 
 class LabelDifference(Metric):
     """
-    ContrastivityCase Measures how different explanations
-    are actually different from each other
+    Obtain explanations for all targets and compare them.
+    In binary case one metric can be computed - the difference
+    between explanations of positive and negative targets.
+    In case of multiclass classification we can compare one explanation
+    to explanations of all other classes and obtain a number of metrics
+    which can be averaged.
+
+    **The more the better**
+
+    - **Worst case:** same explanations for different targets - constant explainer
+    - **Best case:** different explanations for different targets
     """
 
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
diff --git a/xaib/metrics/feature_importance/other_disagreement.py b/xaib/metrics/feature_importance/other_disagreement.py
index 86db5cf..a419ec9 100644
--- a/xaib/metrics/feature_importance/other_disagreement.py
+++ b/xaib/metrics/feature_importance/other_disagreement.py
@@ -9,9 +9,10 @@
 
 class OtherDisagreement(Metric):
     """
-    Coherence measures how method
-    complies with domain knowledge, ground-truth
-    or other methods
+    Measures how distant explanations on the same data points for
+    this particular method from explanations of all others.
+    Average RMSE is used as a metric.
+    **The less the better**
     """
 
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
diff --git a/xaib/metrics/feature_importance/parameter_randomization_check.py b/xaib/metrics/feature_importance/parameter_randomization_check.py
index 5a0cf82..bb92b01 100644
--- a/xaib/metrics/feature_importance/parameter_randomization_check.py
+++ b/xaib/metrics/feature_importance/parameter_randomization_check.py
@@ -9,9 +9,22 @@
 
 class ParameterRandomizationCheck(Metric):
     """
-    CorrectnessCase Measures truthfullness of the method
-    to the underlying model - whether it is
-    sensitive to the changes in model
+    Parameter randomization check is a sanity-check.
+    To ensure that the model influence explanations the
+    following is done. The model is changed and it is expected that
+    explanations should not stay the same is model changed.
+    This check uses random model baselines instead of same models
+    with randomized internal states.
+    Then the explanations on the original data are obtained.
+    They are compared with explanations done with the original model using
+    average RMSE on the whole dataset.
+    The further original explanations from the explanations on
+    the randomized model the better.
+
+    **The less the better**
+     - **Worst case:** explanations are the same, so it is Constant explainer.
+     - **Best case:** is reached when explanations are the opposite, distance between them maximized. The problem with this kind of metric is with its maximization. It seems redundant to maximize it because more different explanations on random states do not mean that the model is more correct.
+    It is difficult to define best case explainer in this case - the metric has no maximum value.
     """
 
     def __init__(
diff --git a/xaib/metrics/feature_importance/small_noise_check.py b/xaib/metrics/feature_importance/small_noise_check.py
index 381f0d6..46e6920 100644
--- a/xaib/metrics/feature_importance/small_noise_check.py
+++ b/xaib/metrics/feature_importance/small_noise_check.py
@@ -9,8 +9,12 @@
 
 class SmallNoiseCheck(Metric):
     """
-    Continuity measures how continuous
-    explanation function is
+    Apply noise of small magnitude to the input data.
+    Obtain original and perturbed explanations.
+    Compare them using RMSE and average.
+    **The less the better**
+     - **Worst case:** is when explanations are hugely changed by the small variations in input
+     - **Best case:** is no variations, so constant explainer should achieve best results
     """
 
     def __init__(
diff --git a/xaib/metrics/feature_importance/sparsity.py b/xaib/metrics/feature_importance/sparsity.py
index 2d6ce70..ffa2847 100644
--- a/xaib/metrics/feature_importance/sparsity.py
+++ b/xaib/metrics/feature_importance/sparsity.py
@@ -9,9 +9,15 @@
 
 class Sparsity(Metric):
     """
-    Compactness measures how compact
-    representations are. Explanations are more
-    understandable if they are short.
+    Considering Gini-index as a measure of sparsity, one can give an
+    average of it as a measure of sparsity for explanations.
+    **The greater the better**
+     - **Worst case:** is achieved by constant explainer that gives same
+     importance to each feature that is equal to 1/N where N is the number
+     of features will obtain best gini index and hence worst sparsity
+     - **Best case:** is when explainer is constant and gives one feature
+     maximum value and others zero, which is the most unequal distribution
+     and is the sparsest explanation that can be given
     """
 
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:

From 858a5c842f890247e76d77599258a975ec6bc455 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sun, 2 Apr 2023 13:27:21 +0300
Subject: [PATCH 14/19] Add notions

---
 xaib/docs/source/datasets.rst | 2 ++
 xaib/docs/source/metrics.rst  | 2 ++
 xaib/docs/source/models.rst   | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/xaib/docs/source/datasets.rst b/xaib/docs/source/datasets.rst
index a84e148..830acd3 100644
--- a/xaib/docs/source/datasets.rst
+++ b/xaib/docs/source/datasets.rst
@@ -1,3 +1,5 @@
 Datasets
 ========
 Coming soon
+
+Until that, consider reading reference on datasets.
diff --git a/xaib/docs/source/metrics.rst b/xaib/docs/source/metrics.rst
index 4ed8d62..847467f 100644
--- a/xaib/docs/source/metrics.rst
+++ b/xaib/docs/source/metrics.rst
@@ -1,3 +1,5 @@
 Metrics
 =======
 Coming soon
+
+Until that, consider reading reference on metrics.
diff --git a/xaib/docs/source/models.rst b/xaib/docs/source/models.rst
index 4b0b3ea..a0d80b1 100644
--- a/xaib/docs/source/models.rst
+++ b/xaib/docs/source/models.rst
@@ -1,3 +1,5 @@
 Models
 ======
 Coming soon
+
+Until that, consider reading reference on models.

From 6b2379882354c158d7f5438374d5e4a1d0ad0844 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sun, 2 Apr 2023 13:34:21 +0300
Subject: [PATCH 15/19] Fix indent...

---
 xaib/metrics/feature_importance/sparsity.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/xaib/metrics/feature_importance/sparsity.py b/xaib/metrics/feature_importance/sparsity.py
index ffa2847..c29a48d 100644
--- a/xaib/metrics/feature_importance/sparsity.py
+++ b/xaib/metrics/feature_importance/sparsity.py
@@ -12,12 +12,13 @@ class Sparsity(Metric):
     Considering Gini-index as a measure of sparsity, one can give an
     average of it as a measure of sparsity for explanations.
     **The greater the better**
-     - **Worst case:** is achieved by constant explainer that gives same
-     importance to each feature that is equal to 1/N where N is the number
-     of features will obtain best gini index and hence worst sparsity
-     - **Best case:** is when explainer is constant and gives one feature
-     maximum value and others zero, which is the most unequal distribution
-     and is the sparsest explanation that can be given
+      - **Worst case:** is achieved by constant explainer that gives same
+      importance to each feature that is equal to 1/N where N is the number
+      of features will obtain best gini index and hence worst sparsity
+      - **Best case:** is when explainer is constant and gives one feature
+      maximum value and others zero, which is the most unequal distribution
+      and is the sparsest explanation that can be given
+
     """
 
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:

From f03197a83d953ddea5ba6e6eb4adc968a2223c4d Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sun, 2 Apr 2023 13:34:33 +0300
Subject: [PATCH 16/19] Fix incorrect paths

---
 xaib/docs/source/xaib/metrics.example_selection.rst  |  8 ++++----
 xaib/docs/source/xaib/metrics.feature_importance.rst | 10 +++++-----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/xaib/docs/source/xaib/metrics.example_selection.rst b/xaib/docs/source/xaib/metrics.example_selection.rst
index c92a891..9d98ac6 100644
--- a/xaib/docs/source/xaib/metrics.example_selection.rst
+++ b/xaib/docs/source/xaib/metrics.example_selection.rst
@@ -1,7 +1,7 @@
 xaib.metrics.example_selection
 ==============================
 
-.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.CovariateRegularity
+.. autoclass:: xaib.metrics.example_selection.covariate_regularity.CovariateRegularity
    :members:
 
 |
@@ -10,7 +10,7 @@ xaib.metrics.example_selection
 
 |
 
-.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.ParameterRandomizationCheck
+.. autoclass:: xaib.metrics.example_selection.parameter_randomization_check.ParameterRandomizationCheck
    :members:
 
 |
@@ -19,7 +19,7 @@ xaib.metrics.example_selection
 
 |
 
-.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.SmallNoiseCheck
+.. autoclass:: xaib.metrics.example_selection.small_noise_check.SmallNoiseCheck
    :members:
 
 |
@@ -28,7 +28,7 @@ xaib.metrics.example_selection
 
 |
 
-.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.TargetDiscriminativeness
+.. autoclass:: xaib.metrics.example_selection.target_discriminativeness.TargetDiscriminativeness
    :members:
 
 |
diff --git a/xaib/docs/source/xaib/metrics.feature_importance.rst b/xaib/docs/source/xaib/metrics.feature_importance.rst
index cb2f047..348597c 100644
--- a/xaib/docs/source/xaib/metrics.feature_importance.rst
+++ b/xaib/docs/source/xaib/metrics.feature_importance.rst
@@ -10,7 +10,7 @@ xaib.metrics.feature_importance
 
 |
 
-.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.LabelDifference
+.. autoclass:: xaib.metrics.feature_importance.label_difference.LabelDifference
    :members:
 
 |
@@ -19,7 +19,7 @@ xaib.metrics.feature_importance
 
 |
 
-.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.OtherDisagreement
+.. autoclass:: xaib.metrics.feature_importance.other_disagreement.OtherDisagreement
    :members:
 
 |
@@ -28,7 +28,7 @@ xaib.metrics.feature_importance
 
 |
 
-.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.ParameterRandomizationCheck
+.. autoclass:: xaib.metrics.feature_importance.parameter_randomization_check.ParameterRandomizationCheck
    :members:
 
 |
@@ -37,7 +37,7 @@ xaib.metrics.feature_importance
 
 |
 
-.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.SmallNoiseCheck
+.. autoclass:: xaib.metrics.feature_importance.small_noise_check.SmallNoiseCheck
    :members:
 
 |
@@ -46,7 +46,7 @@ xaib.metrics.feature_importance
 
 |
 
-.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.Sparsity
+.. autoclass:: xaib.metrics.feature_importance.sparsity.Sparsity
    :members:
 
 |

From be8ede57761d2aa16eb79d18fd61b37e1d41c53b Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sun, 2 Apr 2023 13:34:41 +0300
Subject: [PATCH 17/19] Add missing modules

---
 xaib/docs/source/xaib/datasets.rst | 11 +++++++++++
 xaib/docs/source/xaib/models.rst   |  2 ++
 2 files changed, 13 insertions(+)
 create mode 100644 xaib/docs/source/xaib/datasets.rst
 create mode 100644 xaib/docs/source/xaib/models.rst

diff --git a/xaib/docs/source/xaib/datasets.rst b/xaib/docs/source/xaib/datasets.rst
new file mode 100644
index 0000000..32333ba
--- /dev/null
+++ b/xaib/docs/source/xaib/datasets.rst
@@ -0,0 +1,11 @@
+xaib.datasets
+=============
+
+.. autoclass:: xaib.datasets.synthetic_dataset.SyntheticDataset
+   :members:
+
+|
+
+|
+
+|
diff --git a/xaib/docs/source/xaib/models.rst b/xaib/docs/source/xaib/models.rst
new file mode 100644
index 0000000..805af61
--- /dev/null
+++ b/xaib/docs/source/xaib/models.rst
@@ -0,0 +1,2 @@
+xaib.models
+===========

From 6bb056033cfbc73b1be93db3b16daf92254cc205 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sun, 2 Apr 2023 13:41:11 +0300
Subject: [PATCH 18/19] Fix indent... again

---
 xaib/metrics/feature_importance/covariate_regularity.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/xaib/metrics/feature_importance/covariate_regularity.py b/xaib/metrics/feature_importance/covariate_regularity.py
index 5a2dbb1..d88e351 100644
--- a/xaib/metrics/feature_importance/covariate_regularity.py
+++ b/xaib/metrics/feature_importance/covariate_regularity.py
@@ -16,9 +16,8 @@ class CovariateRegularity(Metric):
     This is measured by average Shannon entropy over batch-normalized explanations.
 
     **The less the better**
-
-      - **Worst case:** constant explainer that gives same importance to each feature, that is equal to 1/N where N is the number of features
-      - **Best case:** constant explainer that gives one feature maximum value and others zero
+     - **Worst case:** constant explainer that gives same importance to each feature, that is equal to 1/N where N is the number of features
+     - **Best case:** constant explainer that gives one feature maximum value and others zero
     """
 
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:

From e66f66ee6c1223a9827967826de887f0786479d0 Mon Sep 17 00:00:00 2001
From: Ilia Moiseev <villeman.5@yandex.ru>
Date: Sun, 2 Apr 2023 13:42:36 +0300
Subject: [PATCH 19/19] Bump version

---
 setup.py         | 20 +++++++-------------
 xaib/__init__.py |  2 +-
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/setup.py b/setup.py
index b24a021..9687b87 100644
--- a/setup.py
+++ b/setup.py
@@ -1,30 +1,24 @@
 import setuptools
 
-
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 setuptools.setup(
     name="xai-benchmark",
-    version='0.2.0-alpha',
-    author='Ilia Moiseev',
-    author_email='ilia.moiseev.5@yandex.ru',
-    license='MIT',
+    version="0.2.0",
+    author="Ilia Moiseev",
+    author_email="ilia.moiseev.5@yandex.ru",
+    license="MIT",
     description="Benchmark for Explainable AI methods",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/oxid15/xai-benchmark",
     classifiers=[
         "Programming Language :: Python :: 3",
-        "Operating System :: OS Independent"
+        "Operating System :: OS Independent",
     ],
     package_dir={"xaib": "./xaib"},
     packages=setuptools.find_packages(),
     python_requires=">=3.8",
-    install_requires=[
-        'cascade-ml>=0.7.2',
-        'scikit-learn',
-        'plotly',
-        'kaleido'
-    ]
-)
\ No newline at end of file
+    install_requires=["cascade-ml>=0.7.2", "scikit-learn", "plotly", "kaleido"],
+)
diff --git a/xaib/__init__.py b/xaib/__init__.py
index 2849723..15478fd 100644
--- a/xaib/__init__.py
+++ b/xaib/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "0.2.0-alpha"
+__version__ = "0.2.0"
 __author__ = "Ilia Moiseev"
 __author_email__ = "ilia.moiseev.5@yandex.ru"