From 3d6696411e98033e9d7570b19e7cafa3e53a37e4 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sun, 16 Apr 2023 13:49:23 +0300 Subject: [PATCH 01/32] Add new datasets to example eval --- xaib/evaluation/example_selection/example_selection.py | 2 +- xaib/evaluation/model_factory.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/xaib/evaluation/example_selection/example_selection.py b/xaib/evaluation/example_selection/example_selection.py index 6a0605e..32bc3d3 100644 --- a/xaib/evaluation/example_selection/example_selection.py +++ b/xaib/evaluation/example_selection/example_selection.py @@ -28,7 +28,7 @@ def __init__(self, *args, blocks=None, name=None, **kwargs) -> None: # Overwrite previous run ModelRepo(REPO_PATH, overwrite=True) -for dataset in ["synthetic_noisy", "synthetic"]: +for dataset in ["synthetic_noisy", "synthetic", "iris", "digits"]: for model in ["knn"]: train_ds, test_ds = DatasetFactory().get(dataset) print(train_ds.get_meta()) diff --git a/xaib/evaluation/model_factory.py b/xaib/evaluation/model_factory.py index ed63220..301c1ea 100644 --- a/xaib/evaluation/model_factory.py +++ b/xaib/evaluation/model_factory.py @@ -13,6 +13,7 @@ def __init__(self, *args, blocks=None, name=None, **kwargs) -> None: self.name = name +# TODO: merge code repetition def svm(train_ds, test_ds): X_train, Y_train = [x["item"] for x in train_ds], [x["label"] for x in train_ds] X_test, Y_test = [x["item"] for x in test_ds], [x["label"] for x in test_ds] @@ -39,7 +40,7 @@ def knn(train_ds, test_ds): model = SkWrapper(blocks=[KNeighborsClassifier(n_neighbors=3)], name="svm") model.fit(X_train, Y_train) - model.evaluate(X_test, Y_test, {"f1": f1_score}) + model.evaluate(X_test, Y_test, {"f1": lambda x, y: f1_score(x, y, average="macro")}) return model From 6b93365e14dff687bad0c6844b157a3ca7d0c3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=BE=D0=B8=D1=81=D0=B5=D0=B5=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=AE=D1=80=D1=8C=D0=B5=D0=B2=D0=B8=D1=87?= Date: Tue, 18 Apr 2023 00:36:04 +0300 Subject: [PATCH 02/32] Add some more new datasets including real-life ones; Restrict eval size --- xaib/datasets/sk_dataset.py | 18 +++++++++++++++++- xaib/evaluation/dataset_factory.py | 9 +++++++++ .../feature_importance/feature_importance.py | 11 +++++++++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/xaib/datasets/sk_dataset.py b/xaib/datasets/sk_dataset.py index c5dec12..0b2bdb7 100644 --- a/xaib/datasets/sk_dataset.py +++ b/xaib/datasets/sk_dataset.py @@ -2,8 +2,21 @@ import numpy as np from cascade import data as cdd -from sklearn.datasets import load_iris, load_digits +from sklearn.datasets import (load_iris, load_digits, load_breast_cancer, fetch_covtype, + fetch_kddcup99) from sklearn.model_selection import train_test_split +from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder + + +def prepare_kddcup99(): + x, y = fetch_kddcup99(return_X_y=True) # 1, 2, 3 - categorical + + enc = OneHotEncoder(sparse_output=False) + cats = enc.fit_transform(x[:, 1:3]) + x = np.concatenate((x[:, 0].reshape(len(x), 1), cats, x[:, 4:]), axis=1) + y = OrdinalEncoder().fit_transform(y.reshape(len(y), 1)) + + return x, y class SkDataset(cdd.SizedDataset): @@ -15,6 +28,9 @@ def __init__(self, name, split, frac=0.8, *args, **kwargs) -> None: constructors = { "iris": load_iris(return_X_y=True), "digits": load_digits(return_X_y=True), + "breast_cancer": load_breast_cancer(return_X_y=True), + "covtype": fetch_covtype(return_X_y=True), + 'kddcup99': prepare_kddcup99() } if name not in constructors: diff --git a/xaib/evaluation/dataset_factory.py b/xaib/evaluation/dataset_factory.py index e958e66..e3acbda 100644 --- a/xaib/evaluation/dataset_factory.py +++ b/xaib/evaluation/dataset_factory.py @@ -43,3 +43,12 @@ def __init__(self) -> None: self._constructors["digits"] = lambda: generate_dataset( SkDataset, "digits", frac=0.8 ) + self._constructors["breast_cancer"] = lambda: generate_dataset( + SkDataset, "breast_cancer", frac=0.8 + ) + self._constructors["covtype"] = lambda: generate_dataset( + SkDataset, "covtype", frac=0.8 + ) + self._constructors["kddcup99"] = lambda: generate_dataset( + SkDataset, "kddcup99", frac=0.8 + ) diff --git a/xaib/evaluation/feature_importance/feature_importance.py b/xaib/evaluation/feature_importance/feature_importance.py index 594d8fc..b6423fe 100644 --- a/xaib/evaluation/feature_importance/feature_importance.py +++ b/xaib/evaluation/feature_importance/feature_importance.py @@ -1,6 +1,9 @@ +import matplotlib +matplotlib.use('TkAgg') import os import sys +from cascade import data as cdd from cascade.models import ModelRepo from xaib.evaluation import DatasetFactory, ModelFactory from xaib.evaluation.feature_importance import ExperimentFactory, ExplainerFactory @@ -14,12 +17,13 @@ REPO_PATH = os.path.join(os.path.dirname(BASE_DIR), "results", "feature_importance") BS = 100 - +MAX_EVAL_SAMPLES = 1000 # Overwrite previous run ModelRepo(REPO_PATH, overwrite=True) -for dataset in ["digits", "iris", "synthetic_noisy", "synthetic"]: +for dataset in ["kddcup99", "covtype", "breast_cancer", "digits", + "iris", "synthetic_noisy", "synthetic"]: for model in ["svm"]: train_ds, test_ds = DatasetFactory().get(dataset) print(train_ds.get_meta()) @@ -29,6 +33,9 @@ model = ModelFactory(train_ds, test_ds).get(model) print(model.get_meta()) + if len(test_ds) > MAX_EVAL_SAMPLES: + test_ds = cdd.RandomSampler(test_ds, MAX_EVAL_SAMPLES) + explainers = ExplainerFactory(train_ds, model, labels=labels).get("all") experiment_factory = ExperimentFactory( REPO_PATH, explainers, test_ds, model, labels, BS From 45f51154ab13ede1676ae0c9c7a96107f44c6333 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sat, 29 Apr 2023 13:02:30 +0300 Subject: [PATCH 03/32] upd logo --- xaib/docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xaib/docs/source/conf.py b/xaib/docs/source/conf.py index be18760..dddfa9c 100644 --- a/xaib/docs/source/conf.py +++ b/xaib/docs/source/conf.py @@ -78,7 +78,7 @@ "html_minify": True, "css_minify": True, "nav_title": "XAIB - Open and extensible benchmark for XAI methods", - "logo_icon": "細部さいぶ", # "質", #'細部' + "logo_icon": "細部", # "質", #'細部' } html_sidebars = { From 355cb42726e0fb8038d54710c3dc194403197d11 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sat, 29 Apr 2023 13:05:08 +0300 Subject: [PATCH 04/32] Update requirements --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 846901d..300c33f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -cascade-ml<0.11.0 +cascade-ml scikit-learn plotly kaleido \ No newline at end of file diff --git a/setup.py b/setup.py index 9687b87..8584027 100644 --- a/setup.py +++ b/setup.py @@ -20,5 +20,5 @@ package_dir={"xaib": "./xaib"}, packages=setuptools.find_packages(), python_requires=">=3.8", - install_requires=["cascade-ml>=0.7.2", "scikit-learn", "plotly", "kaleido"], + install_requires=["cascade-ml", "scikit-learn", "plotly", "kaleido"], ) From 155de174aabdafef181b9607c02d957e5da8e434 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sat, 29 Apr 2023 13:15:36 +0300 Subject: [PATCH 05/32] Add new dataset --- xaib/datasets/sk_dataset.py | 15 +++++++++++---- xaib/evaluation/dataset_factory.py | 3 +++ .../feature_importance/feature_importance.py | 19 +++++++++++++++---- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/xaib/datasets/sk_dataset.py b/xaib/datasets/sk_dataset.py index 0b2bdb7..9658e8a 100644 --- a/xaib/datasets/sk_dataset.py +++ b/xaib/datasets/sk_dataset.py @@ -2,14 +2,20 @@ import numpy as np from cascade import data as cdd -from sklearn.datasets import (load_iris, load_digits, load_breast_cancer, fetch_covtype, - fetch_kddcup99) +from sklearn.datasets import ( + load_iris, + load_digits, + load_breast_cancer, + fetch_covtype, + fetch_kddcup99, + fetch_lfw_people, +) from sklearn.model_selection import train_test_split from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder def prepare_kddcup99(): - x, y = fetch_kddcup99(return_X_y=True) # 1, 2, 3 - categorical + x, y = fetch_kddcup99(return_X_y=True) # 1, 2, 3 - categorical enc = OneHotEncoder(sparse_output=False) cats = enc.fit_transform(x[:, 1:3]) @@ -30,7 +36,8 @@ def __init__(self, name, split, frac=0.8, *args, **kwargs) -> None: "digits": load_digits(return_X_y=True), "breast_cancer": load_breast_cancer(return_X_y=True), "covtype": fetch_covtype(return_X_y=True), - 'kddcup99': prepare_kddcup99() + "kddcup99": prepare_kddcup99(), + "lfw_people": fetch_lfw_people(return_X_y=True), } if name not in constructors: diff --git a/xaib/evaluation/dataset_factory.py b/xaib/evaluation/dataset_factory.py index e3acbda..b85da73 100644 --- a/xaib/evaluation/dataset_factory.py +++ b/xaib/evaluation/dataset_factory.py @@ -52,3 +52,6 @@ def __init__(self) -> None: self._constructors["kddcup99"] = lambda: generate_dataset( SkDataset, "kddcup99", frac=0.8 ) + self._constructors["lfw_people"] = lambda: generate_dataset( + SkDataset, "lfw_people", frac=0.8 + ) diff --git a/xaib/evaluation/feature_importance/feature_importance.py b/xaib/evaluation/feature_importance/feature_importance.py index b6423fe..d528aa9 100644 --- a/xaib/evaluation/feature_importance/feature_importance.py +++ b/xaib/evaluation/feature_importance/feature_importance.py @@ -1,5 +1,6 @@ import matplotlib -matplotlib.use('TkAgg') + +matplotlib.use("TkAgg") import os import sys @@ -22,9 +23,19 @@ # Overwrite previous run ModelRepo(REPO_PATH, overwrite=True) -for dataset in ["kddcup99", "covtype", "breast_cancer", "digits", - "iris", "synthetic_noisy", "synthetic"]: - for model in ["svm"]: +for dataset in [ + "lfw_people", + "kddcup99", + "covtype", + "breast_cancer", + "digits", + "iris", + "synthetic_noisy", + "synthetic", +]: + for model in ["svm", "nn"]: + print(dataset, model) + train_ds, test_ds = DatasetFactory().get(dataset) print(train_ds.get_meta()) From c9972d0fa18d8ef88b22bd8dfbd4675e6c867e68 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sat, 29 Apr 2023 13:15:43 +0300 Subject: [PATCH 06/32] Add standard nn model --- xaib/evaluation/model_factory.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/xaib/evaluation/model_factory.py b/xaib/evaluation/model_factory.py index 301c1ea..0de3df6 100644 --- a/xaib/evaluation/model_factory.py +++ b/xaib/evaluation/model_factory.py @@ -2,6 +2,7 @@ from cascade.utils.sk_model import SkModel from sklearn.metrics import f1_score from sklearn.neighbors import KNeighborsClassifier +from sklearn.neural_network import MLPClassifier from sklearn.svm import SVC from ..base import Factory @@ -44,8 +45,24 @@ def knn(train_ds, test_ds): return model +def nn(train_ds, test_ds): + X_train, Y_train = [x["item"] for x in train_ds], [x["label"] for x in train_ds] + X_test, Y_test = [x["item"] for x in test_ds], [x["label"] for x in test_ds] + + X_train = np.array(X_train) + Y_train = np.array(Y_train, dtype=int) + X_test = np.array(X_test) + Y_test = np.array(Y_test, dtype=int) + + model = SkWrapper(blocks=[MLPClassifier()], name="nn") + model.fit(X_train, Y_train) + model.evaluate(X_test, Y_test, {"f1": lambda x, y: f1_score(x, y, average="macro")}) + return model + + class ModelFactory(Factory): def __init__(self, train_ds, test_ds) -> None: super().__init__() self._constructors["svm"] = lambda: svm(train_ds, test_ds) self._constructors["knn"] = lambda: knn(train_ds, test_ds) + self._constructors["nn"] = lambda: nn(train_ds, test_ds) From aff0e45df3ddf238fa29b7e2dac927559e8d9743 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sun, 30 Apr 2023 17:30:51 +0300 Subject: [PATCH 07/32] Write version of a benchmark into meta --- xaib/base/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xaib/base/base.py b/xaib/base/base.py index 43b1846..2e23d78 100644 --- a/xaib/base/base.py +++ b/xaib/base/base.py @@ -1,3 +1,4 @@ +from .. import __version__ as version from typing import Union, List, Dict, Any, Callable from cascade import data as cdd from cascade import models as cdm @@ -95,6 +96,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.name = None self._metric_objs = dict() + self._meta_prefix.update({"xaib_version": version}) def add_metric(self, name: str, metric: Metric) -> None: self._metric_objs[name] = metric From 9bd4d43abeaeeaf0d6e361a0177fe251c35c9e14 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sun, 30 Apr 2023 17:31:14 +0300 Subject: [PATCH 08/32] Fix direction of prc ? --- .../metrics/feature_importance/parameter_randomization_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xaib/metrics/feature_importance/parameter_randomization_check.py b/xaib/metrics/feature_importance/parameter_randomization_check.py index bb92b01..1fed2e6 100644 --- a/xaib/metrics/feature_importance/parameter_randomization_check.py +++ b/xaib/metrics/feature_importance/parameter_randomization_check.py @@ -33,7 +33,7 @@ def __init__( super().__init__(ds, model, **kwargs) self._noisy_model = noisy_model self.name = "parameter_randomization_check" - self.direction = "down" + self.direction = "up" def compute( self, From a489e214b076bf76da7521c1a17e911ed6c48773 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sun, 30 Apr 2023 18:49:54 +0300 Subject: [PATCH 09/32] Restrict datasets to only toy for now --- xaib/datasets/sk_dataset.py | 20 ++------------------ xaib/evaluation/dataset_factory.py | 12 +++--------- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/xaib/datasets/sk_dataset.py b/xaib/datasets/sk_dataset.py index 9658e8a..8ad1824 100644 --- a/xaib/datasets/sk_dataset.py +++ b/xaib/datasets/sk_dataset.py @@ -5,24 +5,10 @@ from sklearn.datasets import ( load_iris, load_digits, + load_wine, load_breast_cancer, - fetch_covtype, - fetch_kddcup99, - fetch_lfw_people, ) from sklearn.model_selection import train_test_split -from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder - - -def prepare_kddcup99(): - x, y = fetch_kddcup99(return_X_y=True) # 1, 2, 3 - categorical - - enc = OneHotEncoder(sparse_output=False) - cats = enc.fit_transform(x[:, 1:3]) - x = np.concatenate((x[:, 0].reshape(len(x), 1), cats, x[:, 4:]), axis=1) - y = OrdinalEncoder().fit_transform(y.reshape(len(y), 1)) - - return x, y class SkDataset(cdd.SizedDataset): @@ -33,11 +19,9 @@ def __init__(self, name, split, frac=0.8, *args, **kwargs) -> None: constructors = { "iris": load_iris(return_X_y=True), + "wine": load_wine(return_X_y=True), "digits": load_digits(return_X_y=True), "breast_cancer": load_breast_cancer(return_X_y=True), - "covtype": fetch_covtype(return_X_y=True), - "kddcup99": prepare_kddcup99(), - "lfw_people": fetch_lfw_people(return_X_y=True), } if name not in constructors: diff --git a/xaib/evaluation/dataset_factory.py b/xaib/evaluation/dataset_factory.py index b85da73..082ced9 100644 --- a/xaib/evaluation/dataset_factory.py +++ b/xaib/evaluation/dataset_factory.py @@ -40,18 +40,12 @@ def __init__(self) -> None: self._constructors["iris"] = lambda: generate_dataset( SkDataset, "iris", frac=0.8 ) + self._constructors["wine"] = lambda: generate_dataset( + SkDataset, "wine", frac=0.8 + ) self._constructors["digits"] = lambda: generate_dataset( SkDataset, "digits", frac=0.8 ) self._constructors["breast_cancer"] = lambda: generate_dataset( SkDataset, "breast_cancer", frac=0.8 ) - self._constructors["covtype"] = lambda: generate_dataset( - SkDataset, "covtype", frac=0.8 - ) - self._constructors["kddcup99"] = lambda: generate_dataset( - SkDataset, "kddcup99", frac=0.8 - ) - self._constructors["lfw_people"] = lambda: generate_dataset( - SkDataset, "lfw_people", frac=0.8 - ) From f54eefb62976915ca55a06eecaf27ed1ef66497b Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sun, 30 Apr 2023 18:50:31 +0300 Subject: [PATCH 10/32] Eval with wine ds --- xaib/evaluation/feature_importance/feature_importance.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/xaib/evaluation/feature_importance/feature_importance.py b/xaib/evaluation/feature_importance/feature_importance.py index d528aa9..aaf306d 100644 --- a/xaib/evaluation/feature_importance/feature_importance.py +++ b/xaib/evaluation/feature_importance/feature_importance.py @@ -18,15 +18,12 @@ REPO_PATH = os.path.join(os.path.dirname(BASE_DIR), "results", "feature_importance") BS = 100 -MAX_EVAL_SAMPLES = 1000 # Overwrite previous run ModelRepo(REPO_PATH, overwrite=True) for dataset in [ - "lfw_people", - "kddcup99", - "covtype", + "wine", "breast_cancer", "digits", "iris", @@ -44,9 +41,6 @@ model = ModelFactory(train_ds, test_ds).get(model) print(model.get_meta()) - if len(test_ds) > MAX_EVAL_SAMPLES: - test_ds = cdd.RandomSampler(test_ds, MAX_EVAL_SAMPLES) - explainers = ExplainerFactory(train_ds, model, labels=labels).get("all") experiment_factory = ExperimentFactory( REPO_PATH, explainers, test_ds, model, labels, BS From d60e90d296f37c9402b1ea9ddbb9638c804121b8 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Sun, 30 Apr 2023 18:55:06 +0300 Subject: [PATCH 11/32] Write model params and metric into meta --- xaib/base/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xaib/base/base.py b/xaib/base/base.py index 2e23d78..fed77a9 100644 --- a/xaib/base/base.py +++ b/xaib/base/base.py @@ -83,6 +83,8 @@ def evaluate( self.params["direction"] = self.direction self.params["dataset"] = self._ds.name self.params["model"] = self._model.name + self.params["model_params"] = self._model.params + self.params["model_metrics"] = self._model.metrics self.metrics[self.name] = value From d46428c09dc96fd8441385126ae0a98392f74f92 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Mon, 1 May 2023 18:44:40 +0300 Subject: [PATCH 12/32] Fix knn name typo --- xaib/evaluation/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xaib/evaluation/model_factory.py b/xaib/evaluation/model_factory.py index 0de3df6..78bd7d3 100644 --- a/xaib/evaluation/model_factory.py +++ b/xaib/evaluation/model_factory.py @@ -39,7 +39,7 @@ def knn(train_ds, test_ds): X_test = np.array(X_test) Y_test = np.array(Y_test, dtype=int) - model = SkWrapper(blocks=[KNeighborsClassifier(n_neighbors=3)], name="svm") + model = SkWrapper(blocks=[KNeighborsClassifier(n_neighbors=3)], name="knn") model.fit(X_train, Y_train) model.evaluate(X_test, Y_test, {"f1": lambda x, y: f1_score(x, y, average="macro")}) return model From bfa74678cd167bc00a38e90c425c1fe4cb5059db Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Mon, 1 May 2023 19:05:27 +0300 Subject: [PATCH 13/32] Let factory return all its names --- xaib/base/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/xaib/base/base.py b/xaib/base/base.py index fed77a9..196d01a 100644 --- a/xaib/base/base.py +++ b/xaib/base/base.py @@ -73,7 +73,7 @@ def evaluate( expl: Explainer, batch_size: int = 1, expl_kwargs: Union[Dict[Any, Any], None] = None, - **kwargs: Any + **kwargs: Any, ) -> None: if expl_kwargs is None: expl_kwargs = {} @@ -108,7 +108,7 @@ def evaluate( name: str, expl: Explainer, metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None, - **kwargs: Any + **kwargs: Any, ) -> None: if metrics_kwargs is None: metrics_kwargs = {name: {} for _ in self._metric_objs} @@ -162,3 +162,6 @@ def add( ) -> None: self._constructors[name] = constructor self._constructors_kwargs[name] = constr_kwargs + + def get_names(self): + return list(self._constructors.keys()) From 2c55ecd51a91abb3b5feffa3fcbe2ddec52e351c Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Mon, 1 May 2023 19:06:36 +0300 Subject: [PATCH 14/32] Add factories now can be initialized with empty args --- xaib/evaluation/example_selection/case_factory.py | 2 +- xaib/evaluation/example_selection/experiment_factory.py | 4 +++- xaib/evaluation/feature_importance/case_factory.py | 4 +++- xaib/evaluation/feature_importance/experiment_factory.py | 8 +++++++- xaib/evaluation/feature_importance/explainer_factory.py | 4 +++- xaib/evaluation/model_factory.py | 2 +- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/xaib/evaluation/example_selection/case_factory.py b/xaib/evaluation/example_selection/case_factory.py index 29c6f37..559576d 100644 --- a/xaib/evaluation/example_selection/case_factory.py +++ b/xaib/evaluation/example_selection/case_factory.py @@ -20,7 +20,7 @@ def correctness(test_ds, model): class CaseFactory(Factory): - def __init__(self, test_ds: Dataset, model: Model) -> None: + def __init__(self, test_ds: Dataset = None, model: Model = None) -> None: super().__init__() self._constructors["continuity"] = lambda: continuity(test_ds, model) self._constructors["contrastivity"] = lambda: ContrastivityCase(test_ds, model) diff --git a/xaib/evaluation/example_selection/experiment_factory.py b/xaib/evaluation/example_selection/experiment_factory.py index 6800bdf..de2a4d9 100644 --- a/xaib/evaluation/example_selection/experiment_factory.py +++ b/xaib/evaluation/example_selection/experiment_factory.py @@ -5,7 +5,9 @@ class ExperimentFactory(Factory): - def __init__(self, repo_path, explainers, test_ds, model, batch_size) -> None: + def __init__( + self, repo_path=None, explainers=None, test_ds=None, model=None, batch_size=None + ) -> None: super().__init__() case_factory = CaseFactory(test_ds, model) diff --git a/xaib/evaluation/feature_importance/case_factory.py b/xaib/evaluation/feature_importance/case_factory.py index a2a6715..3625a16 100644 --- a/xaib/evaluation/feature_importance/case_factory.py +++ b/xaib/evaluation/feature_importance/case_factory.py @@ -21,7 +21,9 @@ def continuity(test_ds, model): class CaseFactory(Factory): - def __init__(self, test_ds: Dataset, model: Model, labels) -> None: + def __init__( + self, test_ds: Dataset = None, model: Model = None, labels=None + ) -> None: super().__init__() self._constructors["correctness"] = lambda: correctness( test_ds, model, labels=labels diff --git a/xaib/evaluation/feature_importance/experiment_factory.py b/xaib/evaluation/feature_importance/experiment_factory.py index 063f048..504813d 100644 --- a/xaib/evaluation/feature_importance/experiment_factory.py +++ b/xaib/evaluation/feature_importance/experiment_factory.py @@ -6,7 +6,13 @@ class ExperimentFactory(Factory): def __init__( - self, repo_path, explainers, test_ds, model, labels, batch_size + self, + repo_path=None, + explainers=None, + test_ds=None, + model=None, + labels=None, + batch_size=None, ) -> None: super().__init__() diff --git a/xaib/evaluation/feature_importance/explainer_factory.py b/xaib/evaluation/feature_importance/explainer_factory.py index 4466023..0461346 100644 --- a/xaib/evaluation/feature_importance/explainer_factory.py +++ b/xaib/evaluation/feature_importance/explainer_factory.py @@ -6,7 +6,9 @@ class ExplainerFactory(Factory): - def __init__(self, train_ds: Dataset, model: Model, labels=[0, 1]) -> None: + def __init__( + self, train_ds: Dataset = None, model: Model = None, labels=None + ) -> None: super().__init__() self._constructors["const"] = lambda: ConstantExplainer(constant=1) self._constructors["random"] = lambda: RandomExplainer(shift=-15, magnitude=10) diff --git a/xaib/evaluation/model_factory.py b/xaib/evaluation/model_factory.py index 78bd7d3..c2c0bdc 100644 --- a/xaib/evaluation/model_factory.py +++ b/xaib/evaluation/model_factory.py @@ -61,7 +61,7 @@ def nn(train_ds, test_ds): class ModelFactory(Factory): - def __init__(self, train_ds, test_ds) -> None: + def __init__(self, train_ds=None, test_ds=None) -> None: super().__init__() self._constructors["svm"] = lambda: svm(train_ds, test_ds) self._constructors["knn"] = lambda: knn(train_ds, test_ds) From ceb3808dfeea80d51ddef1b3baf3b0d082b4ba97 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Mon, 1 May 2023 19:06:50 +0300 Subject: [PATCH 15/32] Handle errors while creating objects with message --- xaib/base/base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/xaib/base/base.py b/xaib/base/base.py index 196d01a..beac181 100644 --- a/xaib/base/base.py +++ b/xaib/base/base.py @@ -150,9 +150,12 @@ def _get(self, name: str) -> Any: return constructor(**kwargs) def get(self, name: str) -> Union[Dict[str, Any], Any]: - if name == "all": - return self._get_all() - return self._get(name) + try: + if name == "all": + return self._get_all() + return self._get(name) + except Exception as e: + raise RuntimeError(f"Failed to create object {name} in {self}") from e def add( self, From 55e69696ecb1a1470d1ed52e57c04644d482cd4a Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Mon, 1 May 2023 19:07:09 +0300 Subject: [PATCH 16/32] Add simple Setup class --- xaib/evaluation/utils.py | 93 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/xaib/evaluation/utils.py b/xaib/evaluation/utils.py index 9c9c332..60c543c 100644 --- a/xaib/evaluation/utils.py +++ b/xaib/evaluation/utils.py @@ -1,5 +1,5 @@ import os -from typing import Any +from typing import Any, Union, Literal, List import numpy as np import pandas as pd @@ -9,6 +9,95 @@ from plotly import express as px from plotly import graph_objects as go from scipy.special import softmax +from cascade.base import Traceable +from ..base import Factory + + +class Setup(Traceable): + def __init__( + self, + dataset_factory: Factory, + model_factory: Factory, + explainer_factory: Factory, + case_factory: Factory, + datasets: Union[List[str], Literal["all"]] = "all", + models: Union[List[str], Literal["all"]] = "all", + explainers: Union[List[str], Literal["all"]] = "all", + cases: Union[List[str], Literal["all"]] = "all", + datasets_except: Union[List[str], None] = None, + models_except: Union[List[str], None] = None, + explainers_except: Union[List[str], None] = None, + cases_except: Union[List[str], None] = None, + ) -> None: + super().__init__() + + self._cfg = { + "datasets": { + "input": datasets, + "factory": dataset_factory, + "output": None, + "except": datasets_except, + }, + "models": { + "input": models, + "factory": model_factory, + "output": None, + "except": models_except, + }, + "explainers": { + "input": explainers, + "factory": explainer_factory, + "output": None, + "except": explainers_except, + }, + "cases": { + "input": cases, + "factory": case_factory, + "output": None, + "except": cases_except, + }, + } + + for name in self._cfg: + output = self._cfg[name]["input"] + if output == "all": + output = self._cfg[name]["factory"].get_names() + + if self._cfg[name]["except"]: + output = list(set(output) - set(self._cfg[name]["except"])) + + self._cfg[name]["output"] = output + + self.datasets = self._cfg["datasets"]["output"] + self.models = self._cfg["models"]["output"] + self.explainers = self._cfg["explainers"]["output"] + self.cases = self._cfg["cases"]["output"] + + def __repr__(self) -> str: + for name in self._cfg: + if self._cfg[name]["input"] == "all": + rep = "all" + else: + rep = ", ".join(self._cfg[name]["output"]) + + if self._cfg[name]["except"]: + rep += f" except {', '.join(self._cfg[name]['except'])}" + self._cfg[name]["repr"] = rep + + return ( + f"Experiment setup with {self._cfg['datasets']['repr']} Datasets," + f" {self._cfg['models']['repr']} Models," + f" {self._cfg['explainers']['repr']} Explainers," + f" {self._cfg['cases']['repr']} Cases" + ) + + def get_meta(self) -> PipeMeta: + meta = super().get_meta() + meta[0]["datasets"] = self.datasets + meta[0]["models"] = self.models + meta[0]["explainers"] = self.explainers + meta[0]["cases"] = self.cases + return meta class WrapperModel(cdm.ModelModifier): @@ -148,7 +237,7 @@ def scatter(df, metric): y="value", title=metric.replace("_", " ").capitalize() + ", direction - " + direction, hover_data=["dataset", "model"], - color="dataset" + color="dataset", ) From 8c04ce0c8d51148ede62d2e8c497125b312f8492 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Mon, 1 May 2023 19:12:59 +0300 Subject: [PATCH 17/32] Fixes --- xaib/evaluation/example_selection/explainer_factory.py | 2 +- xaib/evaluation/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xaib/evaluation/example_selection/explainer_factory.py b/xaib/evaluation/example_selection/explainer_factory.py index bac5e92..9a5beb3 100644 --- a/xaib/evaluation/example_selection/explainer_factory.py +++ b/xaib/evaluation/example_selection/explainer_factory.py @@ -5,7 +5,7 @@ class ExplainerFactory(Factory): - def __init__(self, train_ds: Dataset, model: Model) -> None: + def __init__(self, train_ds: Dataset = None, model: Model = None) -> None: super().__init__() self._constructors["const"] = lambda: ConstantExplainer( train_ds, train_ds[0]["item"] diff --git a/xaib/evaluation/utils.py b/xaib/evaluation/utils.py index 60c543c..42d500b 100644 --- a/xaib/evaluation/utils.py +++ b/xaib/evaluation/utils.py @@ -9,8 +9,8 @@ from plotly import express as px from plotly import graph_objects as go from scipy.special import softmax -from cascade.base import Traceable -from ..base import Factory +from cascade.base import Traceable, PipeMeta +from xaib.base import Factory class Setup(Traceable): From bb0fd88e48e4d31697e27b1c2a4aec83e3032251 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Mon, 1 May 2023 19:13:08 +0300 Subject: [PATCH 18/32] Evaluation with new setups --- .../example_selection/example_selection.py | 49 +++++++------ .../feature_importance/feature_importance.py | 68 +++++++++++-------- 2 files changed, 65 insertions(+), 52 deletions(-) diff --git a/xaib/evaluation/example_selection/example_selection.py b/xaib/evaluation/example_selection/example_selection.py index 32bc3d3..70b3ec1 100644 --- a/xaib/evaluation/example_selection/example_selection.py +++ b/xaib/evaluation/example_selection/example_selection.py @@ -1,11 +1,14 @@ import os import sys -from cascade import data as cdd from cascade.models import ModelRepo from cascade.utils.sk_model import SkModel from xaib.evaluation import DatasetFactory, ModelFactory -from xaib.evaluation.example_selection import ExperimentFactory, ExplainerFactory +from xaib.evaluation.example_selection import ( + ExperimentFactory, + ExplainerFactory, + CaseFactory, +) SCRIPT_DIR = os.path.dirname(__file__) # xaib/results/... @@ -14,13 +17,7 @@ ) sys.path.append(os.path.abspath(os.path.dirname(SCRIPT_DIR))) -from utils import WrapperModel, visualize_results - - -class SkWrapper(SkModel): - def __init__(self, *args, blocks=None, name=None, **kwargs) -> None: - super().__init__(*args, blocks=blocks, **kwargs) - self.name = name +from utils import Setup, visualize_results BS = 5 @@ -28,21 +25,29 @@ def __init__(self, *args, blocks=None, name=None, **kwargs) -> None: # Overwrite previous run ModelRepo(REPO_PATH, overwrite=True) -for dataset in ["synthetic_noisy", "synthetic", "iris", "digits"]: - for model in ["knn"]: - train_ds, test_ds = DatasetFactory().get(dataset) - print(train_ds.get_meta()) +factories = (DatasetFactory(), ModelFactory(), ExplainerFactory(), CaseFactory()) +setups = [Setup(*factories, models=["knn"])] + +for setup in setups: + for dataset in setup.datasets: + for model in setup.models: + train_ds, test_ds = DatasetFactory().get(dataset) + print(train_ds.get_meta()) + + model = ModelFactory(train_ds, test_ds).get(model) + print(model.get_meta()) - model = ModelFactory(train_ds, test_ds).get(model) - print(model.get_meta()) + explainers = { + explainer: ExplainerFactory(train_ds, model).get(explainer) + for explainer in setup.explainers + } - explainers = ExplainerFactory(train_ds, model).get("all") - experiment_factory = ExperimentFactory( - REPO_PATH, explainers, test_ds, model, BS - ) + experiment_factory = ExperimentFactory( + REPO_PATH, explainers, test_ds, model, BS + ) - experiments = experiment_factory.get("all") - for name in experiments: - experiments[name]() + for case in setup.cases: + experiment = experiment_factory.get(case) + experiment() visualize_results(REPO_PATH, REPO_PATH) diff --git a/xaib/evaluation/feature_importance/feature_importance.py b/xaib/evaluation/feature_importance/feature_importance.py index aaf306d..63ebd1a 100644 --- a/xaib/evaluation/feature_importance/feature_importance.py +++ b/xaib/evaluation/feature_importance/feature_importance.py @@ -1,20 +1,25 @@ import matplotlib matplotlib.use("TkAgg") + import os import sys -from cascade import data as cdd from cascade.models import ModelRepo from xaib.evaluation import DatasetFactory, ModelFactory -from xaib.evaluation.feature_importance import ExperimentFactory, ExplainerFactory +from xaib.evaluation.feature_importance import ( + ExperimentFactory, + ExplainerFactory, + CaseFactory, +) SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_DIR = os.path.dirname(SCRIPT_DIR) PROJECT_DIR = os.path.dirname(os.path.dirname(BASE_DIR)) sys.path.append(PROJECT_DIR) -from xaib.evaluation.utils import visualize_results +from xaib.evaluation.utils import visualize_results, Setup + REPO_PATH = os.path.join(os.path.dirname(BASE_DIR), "results", "feature_importance") BS = 100 @@ -22,32 +27,35 @@ # Overwrite previous run ModelRepo(REPO_PATH, overwrite=True) -for dataset in [ - "wine", - "breast_cancer", - "digits", - "iris", - "synthetic_noisy", - "synthetic", -]: - for model in ["svm", "nn"]: - print(dataset, model) - - train_ds, test_ds = DatasetFactory().get(dataset) - print(train_ds.get_meta()) - - labels = train_ds.labels - - model = ModelFactory(train_ds, test_ds).get(model) - print(model.get_meta()) - - explainers = ExplainerFactory(train_ds, model, labels=labels).get("all") - experiment_factory = ExperimentFactory( - REPO_PATH, explainers, test_ds, model, labels, BS - ) - - experiments = experiment_factory.get("all") - for name in experiments: - experiments[name]() +factories = (DatasetFactory(), ModelFactory(), ExplainerFactory(), CaseFactory()) +setups = [Setup(*factories, models_except="knn")] + +for setup in setups: + for dataset in setup.datasets: + for model in setup.models: + print(dataset, model) + + train_ds, test_ds = DatasetFactory().get(dataset) + print(train_ds.get_meta()) + + labels = train_ds.labels + + model = ModelFactory(train_ds, test_ds).get(model) + print(model.get_meta()) + + explainers = { + explainer: ExplainerFactory(train_ds, model, labels=labels).get( + explainer + ) + for explainer in setup.explainers + } + + experiment_factory = ExperimentFactory( + REPO_PATH, explainers, test_ds, model, labels, BS + ) + + for case in setup.cases: + experiment = experiment_factory.get(case) + experiment() visualize_results(REPO_PATH, REPO_PATH) From 21a3f488d621f98f61a1e8fc0488679bb04f998d Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Mon, 1 May 2023 19:17:25 +0300 Subject: [PATCH 19/32] Write more data in meta of data --- xaib/datasets/sk_dataset.py | 10 ++++++++++ xaib/datasets/synthetic_dataset.py | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/xaib/datasets/sk_dataset.py b/xaib/datasets/sk_dataset.py index 8ad1824..7b44ad7 100644 --- a/xaib/datasets/sk_dataset.py +++ b/xaib/datasets/sk_dataset.py @@ -1,4 +1,5 @@ from typing import Any, Dict +from cascade.base import PipeMeta import numpy as np from cascade import data as cdd @@ -12,10 +13,12 @@ class SkDataset(cdd.SizedDataset): + # TODO: this is classification dataset - should be renamed def __init__(self, name, split, frac=0.8, *args, **kwargs) -> None: super().__init__(**kwargs) self.name = name + self.split = split constructors = { "iris": load_iris(return_X_y=True), @@ -44,3 +47,10 @@ def __len__(self) -> int: def __getitem__(self, index: int) -> Dict[str, Any]: return {"item": self.x[index], "label": self.y[index]} + + def get_meta(self): + meta = super().get_meta() + meta[0]["dataset_name"] = self.name + meta[0]["split"] = self.split + meta[0]["labels"] = self.labels + return meta diff --git a/xaib/datasets/synthetic_dataset.py b/xaib/datasets/synthetic_dataset.py index 70478aa..286caba 100644 --- a/xaib/datasets/synthetic_dataset.py +++ b/xaib/datasets/synthetic_dataset.py @@ -31,3 +31,8 @@ def __len__(self) -> int: def __getitem__(self, index: int) -> Dict[str, Any]: return {"item": self.x[index], "label": self.y[index]} + + def get_meta(self): + meta = super().get_meta() + meta[0]["dataset_name"] = self.name + return meta From d8a1929b5a169701f788d978b84d10f4fb30cc10 Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Thu, 4 May 2023 17:08:52 +0300 Subject: [PATCH 20/32] Conduct final experiments --- xaib/evaluation/dataset_factory.py | 4 ++-- xaib/evaluation/feature_importance/feature_importance.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xaib/evaluation/dataset_factory.py b/xaib/evaluation/dataset_factory.py index 082ced9..497c75f 100644 --- a/xaib/evaluation/dataset_factory.py +++ b/xaib/evaluation/dataset_factory.py @@ -16,7 +16,7 @@ def __init__(self) -> None: super().__init__() self._constructors["synthetic"] = lambda: generate_dataset( SyntheticDataset, - n_samples=100, + n_samples=1000, n_features=14, random_state=0, n_informative=14, @@ -28,7 +28,7 @@ def __init__(self) -> None: self._constructors["synthetic_noisy"] = lambda: generate_dataset( SyntheticDataset, name="synthetic_noisy", - n_samples=100, + n_samples=1000, n_features=14, random_state=0, n_informative=7, diff --git a/xaib/evaluation/feature_importance/feature_importance.py b/xaib/evaluation/feature_importance/feature_importance.py index 63ebd1a..2cf3b95 100644 --- a/xaib/evaluation/feature_importance/feature_importance.py +++ b/xaib/evaluation/feature_importance/feature_importance.py @@ -28,7 +28,7 @@ ModelRepo(REPO_PATH, overwrite=True) factories = (DatasetFactory(), ModelFactory(), ExplainerFactory(), CaseFactory()) -setups = [Setup(*factories, models_except="knn")] +setups = [Setup(*factories, models_except=["knn"])] for setup in setups: for dataset in setup.datasets: From 26f17c1fc8966a0b809412369b5b5eaed9b0c0bc Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Thu, 4 May 2023 17:10:38 +0300 Subject: [PATCH 21/32] Final experiments examples --- xaib/evaluation/example_selection/example_selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xaib/evaluation/example_selection/example_selection.py b/xaib/evaluation/example_selection/example_selection.py index 70b3ec1..0bd3415 100644 --- a/xaib/evaluation/example_selection/example_selection.py +++ b/xaib/evaluation/example_selection/example_selection.py @@ -20,7 +20,7 @@ from utils import Setup, visualize_results -BS = 5 +BS = 100 # Overwrite previous run ModelRepo(REPO_PATH, overwrite=True) From dd99e1294249d73dd457febda529fbe683fae2be Mon Sep 17 00:00:00 2001 From: Ilia Moiseev Date: Thu, 4 May 2023 17:16:13 +0300 Subject: [PATCH 22/32] Update docs --- docs/_modules/index.html | 2 +- docs/_modules/xaib/base/base.html | 26 +++++++--- .../example_selection/continuity_case.html | 2 +- .../feature_importance/coherence_case.html | 2 +- .../feature_importance/compactness_case.html | 2 +- .../feature_importance/continuity_case.html | 2 +- .../contrastivity_case.html | 2 +- .../feature_importance/correctness_case.html | 2 +- .../covariate_complexity_case.html | 2 +- .../xaib/datasets/synthetic_dataset.html | 9 +++- .../example_selection/constant_explainer.html | 2 +- .../example_selection/knn_explainer.html | 2 +- .../example_selection/random_explainer.html | 2 +- .../constant_explainer.html | 2 +- .../feature_importance/lime_explainer.html | 2 +- .../linear_regression_explainer.html | 2 +- .../feature_importance/random_explainer.html | 2 +- .../feature_importance/shap_explainer.html | 2 +- .../covariate_regularity.html | 2 +- .../parameter_randomization_check.html | 2 +- .../example_selection/small_noise_check.html | 2 +- .../target_discriminativeness.html | 2 +- .../covariate_regularity.html | 2 +- .../feature_importance/label_difference.html | 2 +- .../other_disagreement.html | 2 +- .../parameter_randomization_check.html | 4 +- .../feature_importance/small_noise_check.html | 2 +- .../metrics/feature_importance/sparsity.html | 2 +- docs/_modules/xaib/utils/utils.html | 2 +- docs/cases.html | 2 +- docs/contribute.html | 2 +- docs/contribute/add_dataset.html | 2 +- docs/contribute/add_explainer.html | 2 +- docs/contribute/add_metric.html | 2 +- docs/contribute/add_model.html | 2 +- docs/datasets.html | 2 +- docs/genindex.html | 2 +- docs/index.html | 2 +- docs/installation.html | 2 +- docs/metrics.html | 2 +- docs/models.html | 2 +- docs/modules.html | 2 +- docs/objects.inv | Bin 1586 -> 1594 bytes docs/results.html | 2 +- docs/results/example_selection.html | 38 ++++++--------- docs/results/feature_importance.html | 46 +++++++----------- docs/search.html | 2 +- docs/searchindex.js | 2 +- docs/use_cases.html | 2 +- docs/use_cases/evaluate_method.html | 2 +- docs/use_cases/try_method.html | 2 +- docs/xaib/base.html | 2 +- docs/xaib/cases.example_selection.html | 2 +- docs/xaib/cases.feature_importance.html | 2 +- docs/xaib/cases.html | 2 +- docs/xaib/datasets.html | 2 +- docs/xaib/explainers.example_selection.html | 2 +- docs/xaib/explainers.feature_importance.html | 2 +- docs/xaib/explainers.html | 2 +- docs/xaib/metrics.example_selection.html | 2 +- docs/xaib/metrics.feature_importance.html | 2 +- docs/xaib/metrics.html | 2 +- docs/xaib/models.html | 2 +- docs/xaib/utils.html | 2 +- 64 files changed, 117 insertions(+), 122 deletions(-) diff --git a/docs/_modules/index.html b/docs/_modules/index.html index 682c572..81320f4 100644 --- a/docs/_modules/index.html +++ b/docs/_modules/index.html @@ -1 +1 @@ - Overview: module code — xai-benchmark 0.2.0 documentation Skip to content

All modules for which code is available

\ No newline at end of file + Overview: module code — xai-benchmark 0.2.0 documentation Skip to content

All modules for which code is available

\ No newline at end of file diff --git a/docs/_modules/xaib/base/base.html b/docs/_modules/xaib/base/base.html index 74275cf..f9df119 100644 --- a/docs/_modules/xaib/base/base.html +++ b/docs/_modules/xaib/base/base.html @@ -1,5 +1,6 @@ - xaib.base.base — xai-benchmark 0.2.0 documentation Skip to content

Source code for xaib.base.base

-from typing import Union, List, Dict, Any, Callable
+                        xaib.base.base — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.base.base
+from .. import __version__ as version
+from typing import Union, List, Dict, Any, Callable
 from cascade import data as cdd
 from cascade import models as cdm
 
@@ -73,7 +74,7 @@
         expl: Explainer,
         batch_size: int = 1,
         expl_kwargs: Union[Dict[Any, Any], None] = None,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
@@ -83,6 +84,8 @@
         self.params["direction"] = self.direction
         self.params["dataset"] = self._ds.name
         self.params["model"] = self._model.name
+        self.params["model_params"] = self._model.params
+        self.params["model_metrics"] = self._model.metrics
         self.metrics[self.name] = value
 
 
@@ -95,7 +98,8 @@
 
[docs]    def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
-        self._metric_objs = dict()

+        self._metric_objs = dict()
+        self._meta_prefix.update({"xaib_version": version})

 
 
[docs]    def add_metric(self, name: str, metric: Metric) -> None:
         self._metric_objs[name] = metric

@@ -105,7 +109,7 @@
         name: str,
         expl: Explainer,
         metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> None:
         if metrics_kwargs is None:
             metrics_kwargs = {name: {} for _ in self._metric_objs}
@@ -147,9 +151,12 @@
         return constructor(**kwargs)
 
     def get(self, name: str) -> Union[Dict[str, Any], Any]:
-        if name == "all":
-            return self._get_all()
-        return self._get(name)
+        try:
+            if name == "all":
+                return self._get_all()
+            return self._get(name)
+        except Exception as e:
+            raise RuntimeError(f"Failed to create object {name} in {self}") from e
 
     def add(
         self,
@@ -159,4 +166,7 @@
     ) -> None:
         self._constructors[name] = constructor
         self._constructors_kwargs[name] = constr_kwargs
+
+    def get_names(self):
+        return list(self._constructors.keys())
\ No newline at end of file diff --git a/docs/_modules/xaib/cases/example_selection/continuity_case.html b/docs/_modules/xaib/cases/example_selection/continuity_case.html index 15d56c0..ec4edf0 100644 --- a/docs/_modules/xaib/cases/example_selection/continuity_case.html +++ b/docs/_modules/xaib/cases/example_selection/continuity_case.html @@ -1,4 +1,4 @@ - xaib.cases.example_selection.continuity_case — xai-benchmark 0.2.0 documentation Skip to content

Source code for xaib.cases.example_selection.continuity_case

+                        xaib.cases.example_selection.continuity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.example_selection.continuity_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/coherence_case.html b/docs/_modules/xaib/cases/feature_importance/coherence_case.html
index bfbf9db..e4b0c65 100644
--- a/docs/_modules/xaib/cases/feature_importance/coherence_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/coherence_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.coherence_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.coherence_case
+                        xaib.cases.feature_importance.coherence_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.coherence_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/compactness_case.html b/docs/_modules/xaib/cases/feature_importance/compactness_case.html
index 37d74b6..6a4cd0f 100644
--- a/docs/_modules/xaib/cases/feature_importance/compactness_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/compactness_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.compactness_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.compactness_case
+                        xaib.cases.feature_importance.compactness_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.compactness_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/continuity_case.html b/docs/_modules/xaib/cases/feature_importance/continuity_case.html
index 232345c..808d6c3 100644
--- a/docs/_modules/xaib/cases/feature_importance/continuity_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/continuity_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.continuity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.continuity_case
+                        xaib.cases.feature_importance.continuity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.continuity_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html b/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
index 8f82247..3a4e0e2 100644
--- a/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.contrastivity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.contrastivity_case
+                        xaib.cases.feature_importance.contrastivity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.contrastivity_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/correctness_case.html b/docs/_modules/xaib/cases/feature_importance/correctness_case.html
index eba1378..df84884 100644
--- a/docs/_modules/xaib/cases/feature_importance/correctness_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/correctness_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.correctness_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.correctness_case
+                        xaib.cases.feature_importance.correctness_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.correctness_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html b/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
index 052e9d8..cde6c99 100644
--- a/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.covariate_complexity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.covariate_complexity_case
+                        xaib.cases.feature_importance.covariate_complexity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.covariate_complexity_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/datasets/synthetic_dataset.html b/docs/_modules/xaib/datasets/synthetic_dataset.html
index fa51e37..aff3a56 100644
--- a/docs/_modules/xaib/datasets/synthetic_dataset.html
+++ b/docs/_modules/xaib/datasets/synthetic_dataset.html
@@ -1,4 +1,4 @@
-                        xaib.datasets.synthetic_dataset — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.datasets.synthetic_dataset
+                        xaib.datasets.synthetic_dataset — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.datasets.synthetic_dataset
 from typing import Any, Dict
 
 from cascade import data as cdd
@@ -31,5 +31,10 @@
         return len(self.x)
 
     def __getitem__(self, index: int) -> Dict[str, Any]:
-        return {"item": self.x[index], "label": self.y[index]}

+        return {"item": self.x[index], "label": self.y[index]}
+
+
[docs]    def get_meta(self):
+        meta = super().get_meta()
+        meta[0]["dataset_name"] = self.name
+        return meta

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/explainers/example_selection/constant_explainer.html b/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
index fab636a..1ed9cb6 100644
--- a/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
+++ b/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.example_selection.constant_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.constant_explainer
+                        xaib.explainers.example_selection.constant_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.constant_explainer
 import numpy as np
 
 from ...base import Explainer
diff --git a/docs/_modules/xaib/explainers/example_selection/knn_explainer.html b/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
index f056b85..5dba701 100644
--- a/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
+++ b/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.example_selection.knn_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.knn_explainer
+                        xaib.explainers.example_selection.knn_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.knn_explainer
 import numpy as np
 
 from ...base import Explainer
diff --git a/docs/_modules/xaib/explainers/example_selection/random_explainer.html b/docs/_modules/xaib/explainers/example_selection/random_explainer.html
index 4e78602..a934bc5 100644
--- a/docs/_modules/xaib/explainers/example_selection/random_explainer.html
+++ b/docs/_modules/xaib/explainers/example_selection/random_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.example_selection.random_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.random_explainer
+                        xaib.explainers.example_selection.random_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.random_explainer
 import numpy as np
 
 from ...base import Explainer
diff --git a/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html b/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
index 4f9cb4a..5014968 100644
--- a/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.constant_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.constant_explainer
+                        xaib.explainers.feature_importance.constant_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.constant_explainer
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html b/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
index 29455d3..2700097 100644
--- a/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.lime_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.lime_explainer
+                        xaib.explainers.feature_importance.lime_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.lime_explainer
 import numpy as np
 from lime import lime_tabular
 
diff --git a/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html b/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
index a8d4835..00cf00a 100644
--- a/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.linear_regression_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.linear_regression_explainer
+                        xaib.explainers.feature_importance.linear_regression_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.linear_regression_explainer
 import numpy as np
 
 from ...base import Explainer
diff --git a/docs/_modules/xaib/explainers/feature_importance/random_explainer.html b/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
index c253074..d1c2825 100644
--- a/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.random_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.random_explainer
+                        xaib.explainers.feature_importance.random_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.random_explainer
 from typing import Any
 
 import numpy as np
diff --git a/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html b/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
index 31a172a..454f305 100644
--- a/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.shap_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.shap_explainer
+                        xaib.explainers.feature_importance.shap_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.shap_explainer
 import numpy as np
 import shap
 
diff --git a/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html b/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html
index 8ca5981..6907164 100644
--- a/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html
+++ b/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.example_selection.covariate_regularity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.covariate_regularity
+                        xaib.metrics.example_selection.covariate_regularity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.covariate_regularity
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html b/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html
index 5ce0bc2..f3cfa1e 100644
--- a/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html
+++ b/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.example_selection.parameter_randomization_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.parameter_randomization_check
+                        xaib.metrics.example_selection.parameter_randomization_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.parameter_randomization_check
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/example_selection/small_noise_check.html b/docs/_modules/xaib/metrics/example_selection/small_noise_check.html
index eba1101..223979f 100644
--- a/docs/_modules/xaib/metrics/example_selection/small_noise_check.html
+++ b/docs/_modules/xaib/metrics/example_selection/small_noise_check.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.example_selection.small_noise_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.small_noise_check
+                        xaib.metrics.example_selection.small_noise_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.small_noise_check
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html b/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html
index 6f3b3f3..8de255a 100644
--- a/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html
+++ b/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.example_selection.target_discriminativeness — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.target_discriminativeness
+                        xaib.metrics.example_selection.target_discriminativeness — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.target_discriminativeness
 from cascade import data as cdd
 from tqdm import tqdm
 
diff --git a/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html b/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html
index 985732a..7e50b68 100644
--- a/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html
+++ b/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.covariate_regularity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.covariate_regularity
+                        xaib.metrics.feature_importance.covariate_regularity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.covariate_regularity
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/feature_importance/label_difference.html b/docs/_modules/xaib/metrics/feature_importance/label_difference.html
index 16b70ed..168691c 100644
--- a/docs/_modules/xaib/metrics/feature_importance/label_difference.html
+++ b/docs/_modules/xaib/metrics/feature_importance/label_difference.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.label_difference — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.label_difference
+                        xaib.metrics.feature_importance.label_difference — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.label_difference
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html b/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html
index 77e34e1..a9c3190 100644
--- a/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html
+++ b/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.other_disagreement — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.other_disagreement
+                        xaib.metrics.feature_importance.other_disagreement — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.other_disagreement
 from typing import Any, Dict, List, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html b/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html
index 5fb5d17..ead5a4e 100644
--- a/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html
+++ b/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.parameter_randomization_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.parameter_randomization_check
+                        xaib.metrics.feature_importance.parameter_randomization_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.parameter_randomization_check
 from typing import Any, Dict, Union
 
 import numpy as np
@@ -34,7 +34,7 @@
         super().__init__(ds, model, **kwargs)
         self._noisy_model = noisy_model
         self.name = "parameter_randomization_check"
-        self.direction = "down"

+        self.direction = "up"

 
 
[docs]    def compute(
         self,
diff --git a/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html b/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html
index 89b01ac..a46af42 100644
--- a/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html
+++ b/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.small_noise_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.small_noise_check
+                        xaib.metrics.feature_importance.small_noise_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.small_noise_check
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/feature_importance/sparsity.html b/docs/_modules/xaib/metrics/feature_importance/sparsity.html
index 6093569..ff5fa78 100644
--- a/docs/_modules/xaib/metrics/feature_importance/sparsity.html
+++ b/docs/_modules/xaib/metrics/feature_importance/sparsity.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.sparsity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.sparsity
+                        xaib.metrics.feature_importance.sparsity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.sparsity
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/utils/utils.html b/docs/_modules/xaib/utils/utils.html
index 1e1e5db..a91ba0e 100644
--- a/docs/_modules/xaib/utils/utils.html
+++ b/docs/_modules/xaib/utils/utils.html
@@ -1,4 +1,4 @@
-                        xaib.utils.utils — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.utils.utils
+                        xaib.utils.utils — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.utils.utils
 from typing import Dict, List
 
 import numpy as np
diff --git a/docs/cases.html b/docs/cases.html
index 7ab558d..0efdf26 100644
--- a/docs/cases.html
+++ b/docs/cases.html
@@ -1 +1 @@
-                        Cases — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Cases
 
Coming soon
 
 
 
 
 
 
   
\ No newline at end of file
+                        Cases — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Cases
 
Coming soon
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/contribute.html b/docs/contribute.html
index 541726a..02d6623 100644
--- a/docs/contribute.html
+++ b/docs/contribute.html
@@ -1 +1 @@
-                        How to contribute — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
How to contribute
 
Any contributions are welcome! You can help to extend the picture of XAI-methods quality by adding:
 
  
 
 
Note
 
If you want to extend the benchmark by adding new Case, please fill the issue
 
 
 
 
 
 
 
   
\ No newline at end of file
+                        How to contribute — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
How to contribute
 
Any contributions are welcome! You can help to extend the picture of XAI-methods quality by adding:
 
  
 
 
Note
 
If you want to extend the benchmark by adding new Case, please fill the issue
 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/contribute/add_dataset.html b/docs/contribute/add_dataset.html
index 4ebf643..1b55fe3 100644
--- a/docs/contribute/add_dataset.html
+++ b/docs/contribute/add_dataset.html
@@ -1,4 +1,4 @@
-                        Add dataset — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add dataset
 
New datasets may extend our understanding of how different explainers behave in context of different domains and tasks. To add your dataset, you should provide a Wrapper, which will download or access prepared data from disk.
 
 
Create data wrapper
 
First you need to create a wrapper with required interface and fields
 
import numpy as np
+                        Add dataset — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add dataset
 
New datasets may extend our understanding of how different explainers behave in context of different domains and tasks. To add your dataset, you should provide a Wrapper, which will download or access prepared data from disk.
 
 
Create data wrapper
 
First you need to create a wrapper with required interface and fields
 
import numpy as np
 from xaib import Dataset
 
 
diff --git a/docs/contribute/add_explainer.html b/docs/contribute/add_explainer.html
index d00e3ab..7c73955 100644
--- a/docs/contribute/add_explainer.html
+++ b/docs/contribute/add_explainer.html
@@ -1,4 +1,4 @@
-                        Add explainer — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add explainer
 
Explainers are the heart of this benchmark, they are being thorougly tested and the more of them added, the more we know
 
 
Create explainer wrapper
 
Explainers wrappers are less demanding than model’s which makes them easier to be implemented
 
import numpy as np
+                        Add explainer — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add explainer
 
Explainers are the heart of this benchmark, they are being thorougly tested and the more of them added, the more we know
 
 
Create explainer wrapper
 
Explainers wrappers are less demanding than model’s which makes them easier to be implemented
 
import numpy as np
 from xaib import Explainer
 
 
diff --git a/docs/contribute/add_metric.html b/docs/contribute/add_metric.html
index 4d309ef..715ad0d 100644
--- a/docs/contribute/add_metric.html
+++ b/docs/contribute/add_metric.html
@@ -1,4 +1,4 @@
-                        Add metric — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add metric
 
Metrics are ways to numerically assess the quality of explainers and are parts of Cases
 
 
Create metric
 
First you need to create a Metric object - which will accept and explainer and data and return some value
 
 
Note
 
You should define self.name and self.direction of the metric in order for it to be displayed correctly in results. self.name is the short name of what is measured and self.direction denotes what values are considered better - greater or less.
 
 
from xaib import Metric
+                        Add metric — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add metric
 
Metrics are ways to numerically assess the quality of explainers and are parts of Cases
 
 
Create metric
 
First you need to create a Metric object - which will accept and explainer and data and return some value
 
 
Note
 
You should define self.name and self.direction of the metric in order for it to be displayed correctly in results. self.name is the short name of what is measured and self.direction denotes what values are considered better - greater or less.
 
 
from xaib import Metric
 
 class NewMetric(Metric):
     def __init__(self, ds, model *args, **kwargs):
diff --git a/docs/contribute/add_model.html b/docs/contribute/add_model.html
index 22c4e61..4ae5327 100644
--- a/docs/contribute/add_model.html
+++ b/docs/contribute/add_model.html
@@ -1,4 +1,4 @@
-                        Add model — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add model
 
New models and model classes provide information on how good explainers are in some particular cases.
 
 
Create model wrapper
 
First model wrapper should be implemented. It has many required methods that should be implemented. For example fit and evaluate methods are needed to be able to train the model on different datasets see specification in xaib/base and examples in xaib/evaluation/model_factory.py
 
import numpy as np
+                        Add model — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add model
 
New models and model classes provide information on how good explainers are in some particular cases.
 
 
Create model wrapper
 
First model wrapper should be implemented. It has many required methods that should be implemented. For example fit and evaluate methods are needed to be able to train the model on different datasets see specification in xaib/base and examples in xaib/evaluation/model_factory.py
 
import numpy as np
 from xaib import Model
 
 
diff --git a/docs/datasets.html b/docs/datasets.html
index fa47cf3..6dc70c0 100644
--- a/docs/datasets.html
+++ b/docs/datasets.html
@@ -1 +1 @@
-                        Datasets — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Datasets
 
Coming soon
 
Until that, consider reading reference on datasets.
 
 
 
 
 
 
   
\ No newline at end of file
+                        Datasets — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Datasets
 
Coming soon
 
Until that, consider reading reference on datasets.
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/genindex.html b/docs/genindex.html
index 832b5fb..0f151ba 100644
--- a/docs/genindex.html
+++ b/docs/genindex.html
@@ -1 +1 @@
-                        Index — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Index
 
 _ | A | B | C | D | E | K | L | M | O | P | R | S | T 
 
_
 
  
 
A
 
  
 
B
 
   
 
C
 
   
 
D
 
  
 
E
 
   
 
K
 
  
 
L
 
   
 
M
 
   
 
O
 
  
 
P
 
  
 
R
 
  
 
S
 
   
 
T
 
  
 
 
 
 
 
   
\ No newline at end of file
+                        Index — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Index
 
 _ | A | B | C | D | E | G | K | L | M | O | P | R | S | T 
 
_
 
  
 
A
 
  
 
B
 
   
 
C
 
   
 
D
 
  
 
E
 
   
 
G
 
  
 
K
 
  
 
L
 
   
 
M
 
   
 
O
 
  
 
P
 
  
 
R
 
  
 
S
 
   
 
T
 
  
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/index.html b/docs/index.html
index 7083639..61d8d8a 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -1 +1 @@
-                        Welcome to XAIB! — xai-benchmark 0.2.0 documentation                      Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
+                        Welcome to XAIB! — xai-benchmark 0.2.0 documentation                      Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
diff --git a/docs/installation.html b/docs/installation.html
index e8ea378..ae767bc 100644
--- a/docs/installation.html
+++ b/docs/installation.html
@@ -1,4 +1,4 @@
-                        Installation — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Installation
 
To install latest version of a package, run:
 
pip3 install xai-benchmark
+                        Installation — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Installation
 
To install latest version of a package, run:
 
pip3 install xai-benchmark
 
 
 
Remember to create virtual environment if you need one.
 
After the installation you can verify the package by printing out its version:
 
import xaib
 print(xaib.__version__)
 
 
 
To use all explainers you should also install explainers_requirements.txt which can be done directly
 
pip3 install -r https://raw.githubusercontent.com/oxid15/xai-benchmark/master/explainers_requirements.txt
diff --git a/docs/metrics.html b/docs/metrics.html
index 88a3a30..bcaf65c 100644
--- a/docs/metrics.html
+++ b/docs/metrics.html
@@ -1 +1 @@
-                        Metrics — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Metrics
 
Coming soon
 
Until that, consider reading reference on metrics.
 
 
 
 
 
 
   
\ No newline at end of file
+                        Metrics — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Metrics
 
Coming soon
 
Until that, consider reading reference on metrics.
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/models.html b/docs/models.html
index 8b112d4..f79d972 100644
--- a/docs/models.html
+++ b/docs/models.html
@@ -1 +1 @@
-                        Models — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Models
 
Coming soon
 
Until that, consider reading reference on models.
 
 
 
 
 
 
   
\ No newline at end of file
+                        Models — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Models
 
Coming soon
 
Until that, consider reading reference on models.
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/modules.html b/docs/modules.html
index 6dce53a..4e00660 100644
--- a/docs/modules.html
+++ b/docs/modules.html
@@ -1 +1 @@
-                        XAIB modules reference — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
XAIB modules reference
 
  
 
 
 
 
 
 
   
\ No newline at end of file
+                        XAIB modules reference — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
XAIB modules reference
 
  
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/objects.inv b/docs/objects.inv
index 816eb05321917e7d41137add88d5bf95d1807f5d..afb5ad30bbb2279bf8c028978ed0cf299ad63959 100644
GIT binary patch
delta 1030
zcmV+h1o`{247v=kLjr#m06!uo-ud8Ya8&Q7ExSj`L&~B?OAJ*hvcHEp5f^su2wn7T$tR$XYuacNWR+E_==n0-R27CUvM*LlFSBbroz#?&1XG{Hi=KB4Q
z&&e|7zuC2YE88`)PW0|h}T6q>F{?bNbR5q=he$L85f28Nt^j{Kl|uMscUi7!ulCPIJy{(>@;v+z$LdekIdqCUp4
z=g5zu;5GWoZ=rq&8_#+2xQ&`aYx+jYt5?s-SYR_X%RL_qRIEe$>W7~o-+J*K)Q-RT
zK`a?gX}HH?S0oPa(gG$uA0(987*j_K>>vlo;!_kCrTEPJudsY8bGbhoYkTBQr`VB9
zo%)WqmV8W#SeUCbY+)IQ
zBWeodWgG*Oyq)PYq;=h0unpm^jihIA+X>UJcx)0)=ud
zRm#ueCur3wL`~ywl=4!6M6Uk%^!)2jT$%FWkjh2Sv}xrvcq;)_S_Dr3Sgkb8Y&}K
zCt%yIu)dOO3)9@w)@{wp+d*@A8?^U>I_iE}*V`*Xz;(0P0$~{%t{c$6d2h+$$2LZs
z7o31`p{IjvRy5-Z5xDut-?7!c|F_ny5DXsmyAWMLb*X{jqY0N=XN;YCK;P4KTo@Ht
z-$MfTr!OmzD?U~oFtG2mdUJ-wu2lK!EHs)}$5Bg8yG!brw0ozZ-8+H*0YB2@%C@8o
A@&Et;

delta 1022
zcmV)XeCx$`P&@wS2eD*0rQsfn
zU6DAvOADCve2`FTV@w?}u!9^Ri%(Hpl;Sh4)AO%Cab?PfLn;?R)25Zz;H?ByWkt8BD~(kCq8@_Rz<``(81KwV*ykh01Q-Wxjn%SyE%H~O3
zB^Q68tmL)Hw)4K(1G4RZDGi>KIM_F~yBqXwb?<|>tO3DqN*+A7*0lqQAc;?AsFX~Z
z?p6CMJR(q1bt%k8NX6Z*TDc*X4F&3xs88xGpIJ=hY&MAKMsl-a!J!g`N(w
zS<#FuMBwHlf5%q){@+@+LNIvL??QA1)ujf8k0xAhoiTRm0ew%`abZ+oeGduTpT4X>
suJ~AWz`(xK>dhGzyHe$^v(RW_9Y-xW?JlWf((avtcJBoK2W8>HOdcfy;s5{u

diff --git a/docs/results.html b/docs/results.html
index f0005aa..d99e7e6 100644
--- a/docs/results.html
+++ b/docs/results.html
@@ -1 +1 @@
-                        Evaluation results — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
+                        Evaluation results — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
diff --git a/docs/results/example_selection.html b/docs/results/example_selection.html
index 823cd94..227667f 100644
--- a/docs/results/example_selection.html
+++ b/docs/results/example_selection.html
@@ -68,12 +68,7 @@
       

        
       

@@ -169,12 +164,7 @@