diff --git a/docs/_modules/index.html b/docs/_modules/index.html index 953d54d..797e01b 100644 --- a/docs/_modules/index.html +++ b/docs/_modules/index.html @@ -1 +1 @@ - Overview: module code — xai-benchmark 0.2.0-alpha documentation Skip to content

All modules for which code is available

\ No newline at end of file + Overview: module code — xai-benchmark 0.2.0-alpha documentation Skip to content

All modules for which code is available

\ No newline at end of file diff --git a/docs/_modules/xaib/base/base.html b/docs/_modules/xaib/base/base.html index f666666..f8f6afd 100644 --- a/docs/_modules/xaib/base/base.html +++ b/docs/_modules/xaib/base/base.html @@ -1,4 +1,4 @@ - xaib.base.base — xai-benchmark 0.2.0-alpha documentation Skip to content

Source code for xaib.base.base

+                        xaib.base.base — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.base.base
 from typing import Union, List, Dict, Any, Callable
 from cascade import data as cdd
 from cascade import models as cdm
@@ -9,6 +9,7 @@
     Dataset is a wrapper around any collection of items to put in the ML model
     for inference
     """
+
 
[docs]    def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None

@@ -22,6 +23,7 @@
     Model is a wrapper around any inference of ML or other solution in the form y = f(x)
     it implements method `predict` that given certain data x returns the response y
     """
+
 
[docs]    def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None

@@ -35,6 +37,7 @@
     Explainer is a special kind of Model e = g(f, x) that accepts another Model and data as input
     and also returns a response e - an explanation
     """
+
 
[docs]    def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None

@@ -48,6 +51,7 @@
     Metric is an entity which accepts Explainer, Model and Dataset and outputs a metric
     corresponding to the quality of Explainer v = m(g, f, x)
     """
+
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -55,25 +59,30 @@
         self._ds = ds
         self._model = model
 
-    def compute(self, expl: Explainer, batch_size: int = 1, expl_kwargs: Union[Dict[Any, Any], None] = None) -> Any:
+    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> Any:
         raise NotImplementedError()
 
     def evaluate(
-            self,
-            name: str,
-            expl: Explainer,
-            batch_size: int = 1,
-            expl_kwargs: Union[Dict[Any, Any], None] = None,
-            **kwargs: Any
+        self,
+        name: str,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+        **kwargs: Any
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
         value = self.compute(expl, batch_size=batch_size, **expl_kwargs, **kwargs)
 
-        self.params['name'] = name
-        self.params['direction'] = self.direction
-        self.params['dataset'] = self._ds.name
-        self.params['model'] = self._model.name
+        self.params["name"] = name
+        self.params["direction"] = self.direction
+        self.params["dataset"] = self._ds.name
+        self.params["model"] = self._model.name
         self.metrics[self.name] = value
 
 
@@ -82,6 +91,7 @@
     Case is a collection of Metrics which represent some
     high-level property of an Explainer
     """
+
 
[docs]    def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -90,11 +100,17 @@
 
[docs]    def add_metric(self, name: str, metric: Metric) -> None:
         self._metric_objs[name] = metric

 
-
[docs]    def evaluate(self, name: str, expl: Explainer, metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None, **kwargs: Any) -> None:
+
[docs]    def evaluate(
+        self,
+        name: str,
+        expl: Explainer,
+        metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None,
+        **kwargs: Any
+    ) -> None:
         if metrics_kwargs is None:
             metrics_kwargs = {name: {} for _ in self._metric_objs}
 
-        self.params['metric_params'] = dict()
+        self.params["metric_params"] = dict()
         for m_name in self._metric_objs:
             mkwargs = {}
             if m_name in metrics_kwargs:
@@ -102,9 +118,9 @@
 
             self._metric_objs[m_name].evaluate(name, expl, **mkwargs, **kwargs)
 
-            self.params['case'] = self.name
+            self.params["case"] = self.name
 
-            self.params['metric_params'][m_name] = (self._metric_objs[m_name].params)
+            self.params["metric_params"][m_name] = self._metric_objs[m_name].params
             self.metrics.update(self._metric_objs[m_name].metrics)

 
 
@@ -113,15 +129,13 @@
     Collection of constructors to build and return objects
     from predefined hardcoded or added dynamically constructors
     """
+
     def __init__(self) -> None:
         self._constructors = dict()
         self._constructors_kwargs = dict()
 
     def _get_all(self) -> Dict[str, Any]:
-        return {
-                name: self._get(name)
-                for name in self._constructors
-            }
+        return {name: self._get(name) for name in self._constructors}
 
     def _get(self, name: str) -> Any:
         if name not in self._constructors_kwargs:
@@ -133,11 +147,16 @@
         return constructor(**kwargs)
 
     def get(self, name: str) -> Union[Dict[str, Any], Any]:
-        if name == 'all':
+        if name == "all":
             return self._get_all()
         return self._get(name)
 
-    def add(self, name: str, constructor: Callable[[Any], Any], constr_kwargs: Union[Any, None] = None) -> None:
+    def add(
+        self,
+        name: str,
+        constructor: Callable[[Any], Any],
+        constr_kwargs: Union[Any, None] = None,
+    ) -> None:
         self._constructors[name] = constructor
         self._constructors_kwargs[name] = constr_kwargs
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/cases/example_selection/continuity_case.html b/docs/_modules/xaib/cases/example_selection/continuity_case.html
index a9f7b6f..e553222 100644
--- a/docs/_modules/xaib/cases/example_selection/continuity_case.html
+++ b/docs/_modules/xaib/cases/example_selection/continuity_case.html
@@ -1,12 +1,15 @@
-                        xaib.cases.example_selection.continuity_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.example_selection.continuity_case
+                        xaib.cases.example_selection.continuity_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.example_selection.continuity_case
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.example_selection import SmallNoiseCheck
 
 
 
[docs]class ContinuityCase(Case):
-
[docs]    def __init__(self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+
[docs]    def __init__(
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'continuity'
-        self._metric_objs['small_noise_check'] = SmallNoiseCheck(ds, noisy_ds, model)

+        self.name = "continuity"
+        self._metric_objs["small_noise_check"] = SmallNoiseCheck(ds, noisy_ds, model)

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/cases/feature_importance/coherence_case.html b/docs/_modules/xaib/cases/feature_importance/coherence_case.html
index f566228..4867e34 100644
--- a/docs/_modules/xaib/cases/feature_importance/coherence_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/coherence_case.html
@@ -1,12 +1,13 @@
-                        xaib.cases.feature_importance.coherence_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.coherence_case
+                        xaib.cases.feature_importance.coherence_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.coherence_case
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import OtherDisagreement
 
 
 
[docs]class CoherenceCase(Case):
 
[docs]    def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'coherence'
-        self._metric_objs['other_disagreement'] = OtherDisagreement(ds, model)

+        self.name = "coherence"
+        self._metric_objs["other_disagreement"] = OtherDisagreement(ds, model)

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/cases/feature_importance/compactness_case.html b/docs/_modules/xaib/cases/feature_importance/compactness_case.html
index 55b0892..e51e8c9 100644
--- a/docs/_modules/xaib/cases/feature_importance/compactness_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/compactness_case.html
@@ -1,12 +1,13 @@
-                        xaib.cases.feature_importance.compactness_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.compactness_case
+                        xaib.cases.feature_importance.compactness_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.compactness_case
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import Sparsity
 
 
 
[docs]class CompactnessCase(Case):
 
[docs]    def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'compactness'
-        self._metric_objs['sparsity'] = Sparsity(ds, model)

+        self.name = "compactness"
+        self._metric_objs["sparsity"] = Sparsity(ds, model)

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/cases/feature_importance/continuity_case.html b/docs/_modules/xaib/cases/feature_importance/continuity_case.html
index 8bfb2dd..3b9822c 100644
--- a/docs/_modules/xaib/cases/feature_importance/continuity_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/continuity_case.html
@@ -1,12 +1,15 @@
-                        xaib.cases.feature_importance.continuity_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.continuity_case
+                        xaib.cases.feature_importance.continuity_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.continuity_case
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import SmallNoiseCheck
 
 
 
[docs]class ContinuityCase(Case):
-
[docs]    def __init__(self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+
[docs]    def __init__(
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'continuity'
-        self._metric_objs['small_noise_check'] = SmallNoiseCheck(ds, noisy_ds, model)

+        self.name = "continuity"
+        self._metric_objs["small_noise_check"] = SmallNoiseCheck(ds, noisy_ds, model)

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html b/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
index c78fa37..7280d2d 100644
--- a/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
@@ -1,12 +1,13 @@
-                        xaib.cases.feature_importance.contrastivity_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.contrastivity_case
+                        xaib.cases.feature_importance.contrastivity_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.contrastivity_case
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import LabelDifference
 
 
 
[docs]class ContrastivityCase(Case):
 
[docs]    def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'contrastivity'
-        self._metric_objs['label_difference'] = LabelDifference(ds, model)

+        self.name = "contrastivity"
+        self._metric_objs["label_difference"] = LabelDifference(ds, model)

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/cases/feature_importance/correctness_case.html b/docs/_modules/xaib/cases/feature_importance/correctness_case.html
index 9ab02a4..2b387af 100644
--- a/docs/_modules/xaib/cases/feature_importance/correctness_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/correctness_case.html
@@ -1,12 +1,17 @@
-                        xaib.cases.feature_importance.correctness_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.correctness_case
+                        xaib.cases.feature_importance.correctness_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.correctness_case
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import ParameterRandomizationCheck
 
 
 
[docs]class CorrectnessCase(Case):
-
[docs]    def __init__(self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any) -> None:
+
[docs]    def __init__(
+        self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'correctness'
-        self._metric_objs['parameter_randomization_check'] = ParameterRandomizationCheck(ds, model, noisy_model)

+        self.name = "correctness"
+        self._metric_objs[
+            "parameter_randomization_check"
+        ] = ParameterRandomizationCheck(ds, model, noisy_model)

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html b/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
index 64a565d..c3eef4d 100644
--- a/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
@@ -1,12 +1,13 @@
-                        xaib.cases.feature_importance.covariate_complexity_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.covariate_complexity_case
+                        xaib.cases.feature_importance.covariate_complexity_case — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.covariate_complexity_case
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import CovariateRegularity
 
 
 
[docs]class CovariateComplexityCase(Case):
 
[docs]    def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'covariate_complexity'
-        self._metric_objs['covariate_regularity'] = CovariateRegularity(ds, model)

+        self.name = "covariate_complexity"
+        self._metric_objs["covariate_regularity"] = CovariateRegularity(ds, model)

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/datasets/synthetic_dataset.html b/docs/_modules/xaib/datasets/synthetic_dataset.html
new file mode 100644
index 0000000..2e3953c
--- /dev/null
+++ b/docs/_modules/xaib/datasets/synthetic_dataset.html
@@ -0,0 +1,32 @@
+                        xaib.datasets.synthetic_dataset — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.datasets.synthetic_dataset
+from typing import Any, Dict
+
+from cascade import data as cdd
+from sklearn.datasets import make_classification
+from sklearn.model_selection import train_test_split
+
+
+
[docs]class SyntheticDataset(cdd.SizedDataset):
+
[docs]    def __init__(self, split, name=None, frac=0.8, *args, **kwargs) -> None:
+        super().__init__(**kwargs)
+        # Useful for different synthetic datasets
+        if name is not None:
+            self.name = name
+        else:
+            self.name = "synthetic"
+
+        x, y = make_classification(*args, **kwargs)
+
+        train_x, test_x, train_y, test_y = train_test_split(x, y, train_size=frac)
+
+        if split == "train":
+            self.x, self.y = train_x, train_y
+        elif split == "test":
+            self.x, self.y = test_x, test_y

+
+    def __len__(self) -> int:
+        return len(self.x)
+
+    def __getitem__(self, index: int) -> Dict[str, Any]:
+        return {"item": self.x[index], "label": self.y[index]}

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/explainers/example_selection/constant_explainer.html b/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
index 9ae8d69..1263bf5 100644
--- a/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
+++ b/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
@@ -1,5 +1,6 @@
-                        xaib.explainers.example_selection.constant_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.constant_explainer
+                        xaib.explainers.example_selection.constant_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.constant_explainer
 import numpy as np
+
 from ...base import Explainer
 
 
diff --git a/docs/_modules/xaib/explainers/example_selection/knn_explainer.html b/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
index f45697c..09b6d16 100644
--- a/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
+++ b/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
@@ -1,6 +1,7 @@
-                        xaib.explainers.example_selection.knn_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.knn_explainer
-from ...base import Explainer
-import numpy as np
+                        xaib.explainers.example_selection.knn_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.knn_explainer
+import numpy as np
+
+from ...base import Explainer
 
 
 
[docs]class KNNExplainer(Explainer):
@@ -10,5 +11,5 @@
 
 
[docs]    def predict(self, x, model):
         _, indices = model._pipeline[0].kneighbors(x, 1)
-        return np.asarray([self._train_ds[i[0]]['item'] for i in indices])

+        return np.asarray([self._train_ds[i[0]]["item"] for i in indices])

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/explainers/example_selection/random_explainer.html b/docs/_modules/xaib/explainers/example_selection/random_explainer.html
index bb58155..c753d34 100644
--- a/docs/_modules/xaib/explainers/example_selection/random_explainer.html
+++ b/docs/_modules/xaib/explainers/example_selection/random_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.example_selection.random_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.random_explainer
+                        xaib.explainers.example_selection.random_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.random_explainer
 import numpy as np
 
 from ...base import Explainer
@@ -10,6 +10,10 @@
         self._train_ds = train_ds

 
 
[docs]    def predict(self, x, model):
-        return np.asarray([self._train_ds[np.random.randint(0, len(self._train_ds) - 1)]['item']
-            for _ in range(len(x))])

+        return np.asarray(
+            [
+                self._train_ds[np.random.randint(0, len(self._train_ds) - 1)]["item"]
+                for _ in range(len(x))
+            ]
+        )

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html b/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
index b5a69e2..83715fd 100644
--- a/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
@@ -1,12 +1,13 @@
-                        xaib.explainers.feature_importance.constant_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.constant_explainer
-from typing import Any, Union, Dict
+                        xaib.explainers.feature_importance.constant_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.constant_explainer
+from typing import Any, Dict, Union
+
 import numpy as np
 
-from ...base import Model, Explainer
+from ...base import Explainer, Model
 
 
 
[docs]class ConstantExplainer(Explainer):
-
[docs]    def __init__(self, constant: Any, *args:Any, **kwargs: Any) -> None:
+
[docs]    def __init__(self, constant: Any, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.constant = constant

 
diff --git a/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html b/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
index ac143cf..8c07e96 100644
--- a/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
@@ -1,6 +1,6 @@
-                        xaib.explainers.feature_importance.lime_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.lime_explainer
-from lime import lime_tabular
-import numpy as np
+                        xaib.explainers.feature_importance.lime_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.lime_explainer
+import numpy as np
+from lime import lime_tabular
 
 from ...base import Explainer
 
@@ -9,17 +9,21 @@
 
[docs]    def __init__(self, train_ds, labels, *args, meta_prefix=None, **kwargs) -> None:
         super().__init__(*args, meta_prefix=meta_prefix, **kwargs)
 
-        data = np.array([item['item'] for item in train_ds])
-        self._explainer = lime_tabular.LimeTabularExplainer(data, feature_selection='none')
+        data = np.array([item["item"] for item in train_ds])
+        self._explainer = lime_tabular.LimeTabularExplainer(
+            data, feature_selection="none"
+        )
         self._labels = labels

 
 
[docs]    def predict(self, x, model):
-        if not hasattr(model, 'predict_proba'):
-            raise ValueError('The model should have `predict_proba` method')
+        if not hasattr(model, "predict_proba"):
+            raise ValueError("The model should have `predict_proba` method")
 
         explanations = []
         for item in x:
-            ex = self._explainer.explain_instance(item, model.predict_proba, labels=self._labels)
+            ex = self._explainer.explain_instance(
+                item, model.predict_proba, labels=self._labels
+            )
             predicted_label = np.argmax(ex.predict_proba)
             ex = ex.as_map()[predicted_label]
             # Sorts by the feature order
diff --git a/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html b/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
index 56d74b2..8535d7a 100644
--- a/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.linear_regression_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.linear_regression_explainer
+                        xaib.explainers.feature_importance.linear_regression_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.linear_regression_explainer
 import numpy as np
 
 from ...base import Explainer
diff --git a/docs/_modules/xaib/explainers/feature_importance/random_explainer.html b/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
index fd0abe0..adaec2c 100644
--- a/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
@@ -1,7 +1,9 @@
-                        xaib.explainers.feature_importance.random_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.random_explainer
+                        xaib.explainers.feature_importance.random_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.random_explainer
 from typing import Any
+
 import numpy as np
-from ...base import Model, Explainer
+
+from ...base import Explainer, Model
 
 
 
[docs]class RandomExplainer(Explainer):
diff --git a/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html b/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
index 20072a3..0c97202 100644
--- a/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.shap_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.shap_explainer
+                        xaib.explainers.feature_importance.shap_explainer — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.shap_explainer
 import numpy as np
 import shap
 
@@ -9,7 +9,7 @@
 
[docs]    def __init__(self, train_ds, *args, meta_prefix=None, **kwargs) -> None:
         super().__init__(*args, meta_prefix=meta_prefix, **kwargs)
         self._train_ds = train_ds
-        self._train_data = np.array([item['item'] for item in self._train_ds])

+        self._train_data = np.array([item["item"] for item in self._train_ds])

 
 
[docs]    def predict(self, x, model):
         explainer = shap.Explainer(model.predict, self._train_data)
diff --git a/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html b/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html
new file mode 100644
index 0000000..8be0195
--- /dev/null
+++ b/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html
@@ -0,0 +1,73 @@
+                        xaib.metrics.example_selection.covariate_regularity — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.covariate_regularity
+from typing import Any, Dict, Union
+
+import numpy as np
+from tqdm import tqdm
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import Filter, SimpleDataloader, entropy, minmax_normalize
+
+
+
[docs]class CovariateRegularity(Metric):
+    """
+    Measures how noisy the features in the examples provided
+    by the method.
+
+    More simple explanations are considered better.
+    This is measured by average Shannon entropy over batch-normalized explanations.
+
+    **The less the better**
+
+      - **Worst case:** constant explainer that gives examples with same importance to each feature, that is equal to 1/N where N is the number of features
+      - **Best case:** constant explainer that gives examples with one feature with maximum value and others zero
+    """
+
+
[docs]    def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+        super().__init__(ds, model, *args, **kwargs)
+        self.name = "covariate_regularity"
+        self.direction = "down"

+
+
[docs]    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> None:
+        if expl_kwargs is None:
+            expl_kwargs = {}
+
+        # Obtain all the labels
+        labels = np.asarray(
+            [
+                item["label"]
+                for item in tqdm(self._ds, desc="Obtaining labels", leave=False)
+            ]
+        )
+
+        # Determine how much of an intersection different labels have
+        unique_labels, counts = np.unique(labels, return_counts=True)
+        min_len = np.min(counts)
+
+        # For each label get indexes of its items
+        coords = {u: np.argwhere(labels == u).T[0][:min_len] for u in unique_labels}
+
+        # Obtain all explanations by batches
+        explanations = {u: [] for u in unique_labels}
+        for u in unique_labels:
+            dl = SimpleDataloader(Filter(self._ds, coords[u]), batch_size=batch_size)
+            for batch in tqdm(dl):
+                ex = expl.predict(batch["item"], self._model, **expl_kwargs)
+                ex = minmax_normalize(ex)
+
+                explanations[u] += ex.tolist()
+
+        # Compare explanations of different labels
+        entropies_of_features = {u: [] for u in unique_labels}
+        for u in unique_labels:
+            expls = np.asarray(explanations[u])
+            for f in range(expls.shape[1]):
+                e = entropy(expls[:, f])
+                entropies_of_features[u].append(e)
+
+        return np.nanmean([entropies_of_features[u] for u in entropies_of_features])

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html b/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html
new file mode 100644
index 0000000..419f2a7
--- /dev/null
+++ b/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html
@@ -0,0 +1,68 @@
+                        xaib.metrics.example_selection.parameter_randomization_check — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.parameter_randomization_check
+from typing import Any, Dict, Union
+
+import numpy as np
+from tqdm import tqdm
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_count_eq, minmax_normalize
+
+
+
[docs]class ParameterRandomizationCheck(Metric):
+    """
+    Parameter randomization check is a sanity-check.
+    To ensure that the model influence explanations the
+    following is done. The model is changed and it is expected that
+    explanations should not stay the same is model changed.
+    This check uses random model baselines instead of same models
+    with randomized internal states.
+    Then the explanations on the original data are obtained.
+    They are compared with explanations done with the original model by
+    counting how many examples were the same for same data points.
+    The less  the better.
+
+    **The less the better**
+     - **Worst case:** explanations are the same, so it is Constant explainer
+     - **Best case:** is reached when explanations are the opposite,
+     distance between them maximized.
+    The problem with this kind of metric is
+    with its maximization. It seems redundant to maximize it because more
+    different explanations on random states do not mean that the model is
+    more correct.
+    It is difficult to define best case explainer in this case - the metric has no maximum value.
+    """
+
+
[docs]    def __init__(
+        self, ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any
+    ) -> None:
+        super().__init__(ds, model, **kwargs)
+        self._noisy_model = noisy_model
+        self.name = "parameter_randomization_check"
+        self.direction = "down"

+
+
[docs]    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+        expl_noisy_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> None:
+        if expl_kwargs is None:
+            expl_kwargs = {}
+        if expl_noisy_kwargs is None:
+            expl_noisy_kwargs = {}
+
+        diffs_expl = []
+
+        for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
+            item = batch["item"]
+
+            explanation_batch = expl.predict(item, self._model, **expl_kwargs)
+            noisy_explanation_batch = expl.predict(
+                item, self._noisy_model, **expl_noisy_kwargs
+            )
+
+            diffs_expl += batch_count_eq(explanation_batch, noisy_explanation_batch)
+
+        return sum(diffs_expl) / len(diffs_expl)

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/metrics/example_selection/small_noise_check.html b/docs/_modules/xaib/metrics/example_selection/small_noise_check.html
new file mode 100644
index 0000000..b24f6a5
--- /dev/null
+++ b/docs/_modules/xaib/metrics/example_selection/small_noise_check.html
@@ -0,0 +1,63 @@
+                        xaib.metrics.example_selection.small_noise_check — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.small_noise_check
+from typing import Any, Dict, Union
+
+import numpy as np
+from tqdm import tqdm
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_count_eq
+
+
+
[docs]class SmallNoiseCheck(Metric):
+    """
+    Since the nature of example selection methods is discrete - we choose an example
+    from finite set, the use of RMSE measure may be not appropriate.
+    This means that the metric that is similar to the one that was used in feature
+    importance case is not suitable.
+    More appropriate metric could be the following: take the test dataset and add
+    small amount of noise to the items as was done in the feature importance case.
+    Then count the number of item pairs when the example provided didn't change and
+    divide by the total number of items.
+    This ratio then is how continuous example generator is - if it provides the same
+    examples for slightly changed inputs, then it is continuous.
+
+    **The less the better**
+     - **Worst case:** Constant explainer
+     - **Best case:** Random explainer
+    """
+
+
[docs]    def __init__(
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
+    ) -> None:
+        super().__init__(ds, model, *args, **kwargs)
+        self._noisy_ds = noisy_ds
+        self.name = "small_noise_check"
+        self.direction = "up"

+
+
[docs]    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> None:
+        if expl_kwargs is None:
+            expl_kwargs = {}
+
+        counts = []
+        dls = zip(
+            SimpleDataloader(self._ds, batch_size),
+            SimpleDataloader(self._noisy_ds, batch_size),
+        )
+        for batch, noisy_batch in tqdm(dls):
+            item = batch["item"]
+            noisy_item = noisy_batch["item"]
+
+            explanation_batch = expl.predict(item, self._model, **expl_kwargs)
+            noisy_explanation_batch = expl.predict(
+                noisy_item, self._model, **expl_kwargs
+            )
+
+            counts += batch_count_eq(explanation_batch, noisy_explanation_batch)
+
+        return np.sum(counts) / len(self._ds)

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html b/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html
new file mode 100644
index 0000000..ea43c59
--- /dev/null
+++ b/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html
@@ -0,0 +1,51 @@
+                        xaib.metrics.example_selection.target_discriminativeness — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.target_discriminativeness
+from cascade import data as cdd
+from tqdm import tqdm
+
+from ...base import Metric
+from ...evaluation import ModelFactory
+from ...utils import KeyedComposer, SimpleDataloader
+
+
+
[docs]class TargetDiscriminativeness(Metric):
+    """
+    Given true labels and explanations in form of the examples, train another model to discriminate between labels.
+    The quality of the model on the examples given describes the quality of the explanations.
+    The quality can be measured by any performance metric,
+    but it is better to adopt to imbalanced data and use F1-measure for example.
+
+    **The greater the better**
+
+    **Best case:** examples are descriptive of labels so that the model reaches best performance
+    **Worst case:** constant or random baseline - giving insufficient information to grasp labels
+    """
+
+
[docs]    def __init__(self, ds, model, *args, **kwargs):
+        super().__init__(ds, model, *args, **kwargs)
+        self.name = "target_discriminativeness"
+        self.direction = "up"

+
+
[docs]    def compute(self, explainer, batch_size=1, expl_kwargs=None):
+        if expl_kwargs is None:
+            expl_kwargs = {}
+
+        labels = []
+        explanations = []
+        for batch in tqdm(SimpleDataloader(self._ds, batch_size=batch_size)):
+            item, label = batch["item"], batch["label"]
+
+            explanation_batch = explainer.predict(item, self._model, **expl_kwargs)
+
+            explanations += explanation_batch.tolist()
+            labels += label.tolist()
+
+        ds = KeyedComposer(
+            {"item": cdd.Wrapper(explanations), "label": cdd.Wrapper(labels)}
+        )
+
+        expl_train_ds, expl_test_ds = cdd.split(ds, frac=0.8)
+
+        user_model = ModelFactory(expl_train_ds, expl_test_ds).get("svm")
+
+        return user_model.metrics["f1"]

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html b/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html
new file mode 100644
index 0000000..5d748f6
--- /dev/null
+++ b/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html
@@ -0,0 +1,54 @@
+                        xaib.metrics.feature_importance.covariate_regularity — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.covariate_regularity
+from typing import Any, Dict, Union
+
+import numpy as np
+from tqdm import tqdm
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, entropy, minmax_normalize
+
+
+
[docs]class CovariateRegularity(Metric):
+    """
+    Covariate Regularity using entropy over explanations
+
+    This measures how comprehensible the explanations are in average.
+    More simple explanations are considered better.
+    This is measured by average Shannon entropy over batch-normalized explanations.
+
+    **The less the better**
+     - **Worst case:** constant explainer that gives same importance to each feature, that is equal to 1/N where N is the number of features
+     - **Best case:** constant explainer that gives one feature maximum value and others zero
+    """
+
+
[docs]    def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+        super().__init__(ds, model, *args, **kwargs)
+        self.name = "covariate_regularity"
+        self.direction = "down"

+
+
[docs]    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+    ):
+        if expl_kwargs is None:
+            expl_kwargs = {}
+
+        explanations = []
+        for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
+            item = batch["item"]
+
+            e = expl.predict(item, self._model, **expl_kwargs)
+            e = minmax_normalize(e)
+            explanations += e.tolist()
+
+        explanations = np.array(explanations, dtype=float)
+
+        entropies_of_features = []
+        for f in range(explanations.shape[1]):
+            e = entropy(explanations[:, f])
+            entropies_of_features.append(e)
+
+        return np.nanmean(entropies_of_features)

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/metrics/feature_importance/label_difference.html b/docs/_modules/xaib/metrics/feature_importance/label_difference.html
new file mode 100644
index 0000000..b67945d
--- /dev/null
+++ b/docs/_modules/xaib/metrics/feature_importance/label_difference.html
@@ -0,0 +1,81 @@
+                        xaib.metrics.feature_importance.label_difference — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.label_difference
+from typing import Any, Dict, Union
+
+import numpy as np
+from cascade.data import Sampler
+from tqdm import tqdm
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import Filter, SimpleDataloader, batch_rmse, minmax_normalize
+
+
+
[docs]class LabelDifference(Metric):
+    """
+    Obtain explanations for all targets and compare them.
+    In binary case one metric can be computed - the difference
+    between explanations of positive and negative targets.
+    In case of multiclass classification we can compare one explanation
+    to explanations of all other classes and obtain a number of metrics
+    which can be averaged.
+
+    **The more the better**
+
+    - **Worst case:** same explanations for different targets - constant explainer
+    - **Best case:** different explanations for different targets
+    """
+
+
[docs]    def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+        super().__init__(ds, model, *args, **kwargs)
+        self.name = "label_difference"
+        self.direction = "up"

+
+
[docs]    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> None:
+        if expl_kwargs is None:
+            expl_kwargs = {}
+
+        # Obtain all the labels
+        labels = np.asarray(
+            [
+                item["label"]
+                for item in tqdm(self._ds, desc="Obtaining labels", leave=False)
+            ]
+        )
+
+        # Determine how much of an intersection different labels have
+        unique_labels, counts = np.unique(labels, return_counts=True)
+        min_len = np.min(counts)
+
+        # For each label get indexes of its items
+        coords = {u: np.argwhere(labels == u).T[0][:min_len] for u in unique_labels}
+
+        # Obtain all explanations by batches
+        explanations = {u: [] for u in unique_labels}
+        for u in unique_labels:
+            dl = SimpleDataloader(Filter(self._ds, coords[u]), batch_size=batch_size)
+            for batch in tqdm(dl):
+                ex = expl.predict(batch["item"], self._model, **expl_kwargs)
+                ex = minmax_normalize(ex)
+                explanations[u].append(ex)
+
+        # Compare explanations of different labels
+        diffs = {u: [] for u in unique_labels}
+        for u in unique_labels:
+            for other_label in unique_labels:
+                if u == other_label:
+                    continue
+
+                for batch, other_batch in zip(
+                    explanations[u], explanations[other_label]
+                ):
+                    rmse = batch_rmse(batch, other_batch)
+                    diffs[u] += list(rmse)
+        for u in unique_labels:
+            diffs[u] = np.nanmean(diffs[u])
+
+        return np.nanmean([diffs[u] for u in diffs])

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html b/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html
new file mode 100644
index 0000000..7445d64
--- /dev/null
+++ b/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html
@@ -0,0 +1,57 @@
+                        xaib.metrics.feature_importance.other_disagreement — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.other_disagreement
+from typing import Any, Dict, List, Union
+
+import numpy as np
+from tqdm import tqdm
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_rmse, minmax_normalize
+
+
+
[docs]class OtherDisagreement(Metric):
+    """
+    Measures how distant explanations on the same data points for
+    this particular method from explanations of all others.
+    Average RMSE is used as a metric.
+    **The less the better**
+    """
+
+
[docs]    def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+        super().__init__(ds, model, *args, **kwargs)
+        self.name = "other_disagreement"
+        self.direction = "down"

+
+
[docs]    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expls: Union[List[Explainer], None] = None,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+        expls_kwargs: Union[List[Dict[Any, Any]], None] = None,
+    ) -> None:
+        # Default initialization in case of empty
+        # parameter to not to break default argument order
+        # TODO: can revise this later
+        if expls is None:
+            expls = [expl]
+        if expl_kwargs is None:
+            expl_kwargs = {}
+        if expls_kwargs is None:
+            expls_kwargs = [{} for _ in range(len(expls))]
+
+        diffs = []
+        for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
+            item = batch["item"]
+
+            e = expl.predict(item, self._model, **expl_kwargs)
+            e = minmax_normalize(e)
+            other_e = [
+                minmax_normalize(other_expl.predict(item, self._model, **other_kwargs))
+                for other_expl, other_kwargs in zip(expls, expls_kwargs)
+            ]
+
+            for oe in other_e:
+                diffs += batch_rmse(e, oe)
+
+        return np.nanmean(diffs)

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html b/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html
new file mode 100644
index 0000000..851c397
--- /dev/null
+++ b/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html
@@ -0,0 +1,67 @@
+                        xaib.metrics.feature_importance.parameter_randomization_check — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.parameter_randomization_check
+from typing import Any, Dict, Union
+
+import numpy as np
+from tqdm import tqdm
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_rmse, minmax_normalize
+
+
+
[docs]class ParameterRandomizationCheck(Metric):
+    """
+    Parameter randomization check is a sanity-check.
+    To ensure that the model influence explanations the
+    following is done. The model is changed and it is expected that
+    explanations should not stay the same is model changed.
+    This check uses random model baselines instead of same models
+    with randomized internal states.
+    Then the explanations on the original data are obtained.
+    They are compared with explanations done with the original model using
+    average RMSE on the whole dataset.
+    The further original explanations from the explanations on
+    the randomized model the better.
+
+    **The less the better**
+     - **Worst case:** explanations are the same, so it is Constant explainer.
+     - **Best case:** is reached when explanations are the opposite, distance between them maximized. The problem with this kind of metric is with its maximization. It seems redundant to maximize it because more different explanations on random states do not mean that the model is more correct.
+    It is difficult to define best case explainer in this case - the metric has no maximum value.
+    """
+
+
[docs]    def __init__(
+        self, ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any
+    ) -> None:
+        super().__init__(ds, model, **kwargs)
+        self._noisy_model = noisy_model
+        self.name = "parameter_randomization_check"
+        self.direction = "down"

+
+
[docs]    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+        expl_noisy_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> None:
+        if expl_kwargs is None:
+            expl_kwargs = {}
+        if expl_noisy_kwargs is None:
+            expl_noisy_kwargs = {}
+
+        diffs_expl = []
+
+        for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
+            item = batch["item"]
+
+            explanation_batch = expl.predict(item, self._model, **expl_kwargs)
+            noisy_explanation_batch = expl.predict(
+                item, self._noisy_model, **expl_noisy_kwargs
+            )
+
+            explanation_batch = minmax_normalize(explanation_batch)
+            noisy_explanation_batch = minmax_normalize(noisy_explanation_batch)
+
+            diffs_expl += batch_rmse(explanation_batch, noisy_explanation_batch)
+
+        return np.nanmean(diffs_expl)

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html b/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html
new file mode 100644
index 0000000..bb87ca0
--- /dev/null
+++ b/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html
@@ -0,0 +1,58 @@
+                        xaib.metrics.feature_importance.small_noise_check — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.small_noise_check
+from typing import Any, Dict, Union
+
+import numpy as np
+from tqdm import tqdm
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_rmse, minmax_normalize
+
+
+
[docs]class SmallNoiseCheck(Metric):
+    """
+    Apply noise of small magnitude to the input data.
+    Obtain original and perturbed explanations.
+    Compare them using RMSE and average.
+    **The less the better**
+     - **Worst case:** is when explanations are hugely changed by the small variations in input
+     - **Best case:** is no variations, so constant explainer should achieve best results
+    """
+
+
[docs]    def __init__(
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
+    ) -> None:
+        super().__init__(ds, model, *args, **kwargs)
+        self._noisy_ds = noisy_ds
+        self.name = "small_noise_check"
+        self.direction = "down"

+
+
[docs]    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> None:
+        if expl_kwargs is None:
+            expl_kwargs = {}
+
+        rmses = []
+        dls = zip(
+            SimpleDataloader(self._ds, batch_size),
+            SimpleDataloader(self._noisy_ds, batch_size),
+        )
+        for batch, noisy_batch in tqdm(dls):
+            item = batch["item"]
+            noisy_item = noisy_batch["item"]
+
+            explanation_batch = expl.predict(item, self._model, **expl_kwargs)
+            noisy_explanation_batch = expl.predict(
+                noisy_item, self._model, **expl_kwargs
+            )
+
+            explanation_batch = minmax_normalize(explanation_batch)
+            noisy_explanation_batch = minmax_normalize(noisy_explanation_batch)
+
+            rmses += batch_rmse(explanation_batch, noisy_explanation_batch)
+
+        return np.nanmean(rmses)

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/metrics/feature_importance/sparsity.html b/docs/_modules/xaib/metrics/feature_importance/sparsity.html
new file mode 100644
index 0000000..06b5048
--- /dev/null
+++ b/docs/_modules/xaib/metrics/feature_importance/sparsity.html
@@ -0,0 +1,50 @@
+                        xaib.metrics.feature_importance.sparsity — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.sparsity
+from typing import Any, Dict, Union
+
+import numpy as np
+from tqdm import tqdm
+
+from ...base import Dataset, Explainer, Metric, Model
+from ...utils import SimpleDataloader, batch_gini, minmax_normalize
+
+
+
[docs]class Sparsity(Metric):
+    """
+    Considering Gini-index as a measure of sparsity, one can give an
+    average of it as a measure of sparsity for explanations.
+    **The greater the better**
+      - **Worst case:** is achieved by constant explainer that gives same
+      importance to each feature that is equal to 1/N where N is the number
+      of features will obtain best gini index and hence worst sparsity
+      - **Best case:** is when explainer is constant and gives one feature
+      maximum value and others zero, which is the most unequal distribution
+      and is the sparsest explanation that can be given
+
+    """
+
+
[docs]    def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+        super().__init__(ds, model, *args, **kwargs)
+        self.name = "sparsity"
+        self.direction = "up"

+
+
[docs]    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> None:
+        if expl_kwargs is None:
+            expl_kwargs = {}
+
+        ginis = []
+
+        for batch in tqdm(SimpleDataloader(self._ds, batch_size)):
+            item = batch["item"]
+
+            explanation_batch = expl.predict(item, self._model, **expl_kwargs)
+            explanation_batch = minmax_normalize(explanation_batch)
+
+            ginis += batch_gini(explanation_batch)
+
+        return np.nanmean(ginis)

+
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/utils/utils.html b/docs/_modules/xaib/utils/utils.html
index ae40db5..7386f5f 100644
--- a/docs/_modules/xaib/utils/utils.html
+++ b/docs/_modules/xaib/utils/utils.html
@@ -1,7 +1,8 @@
-                        xaib.utils.utils — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.utils.utils
+                        xaib.utils.utils — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.utils.utils
 from typing import Dict, List
+
 import numpy as np
-from cascade.data import SizedDataset, Composer, Sampler
+from cascade.data import Composer, Sampler, SizedDataset
 
 
 def rmse(x, y):
@@ -71,7 +72,7 @@
             item = self._data[i]
             for ch in self._channels:
                 batch[ch].append(item[ch])
-        
+
         for ch in batch:
             batch[ch] = np.array(batch[ch])
 
diff --git a/docs/_sources/datasets.rst.txt b/docs/_sources/datasets.rst.txt
index a84e148..830acd3 100644
--- a/docs/_sources/datasets.rst.txt
+++ b/docs/_sources/datasets.rst.txt
@@ -1,3 +1,5 @@
 Datasets
 ========
 Coming soon
+
+Until that, consider reading reference on datasets.
diff --git a/docs/_sources/metrics.rst.txt b/docs/_sources/metrics.rst.txt
index 4ed8d62..847467f 100644
--- a/docs/_sources/metrics.rst.txt
+++ b/docs/_sources/metrics.rst.txt
@@ -1,3 +1,5 @@
 Metrics
 =======
 Coming soon
+
+Until that, consider reading reference on metrics.
diff --git a/docs/_sources/models.rst.txt b/docs/_sources/models.rst.txt
index 4b0b3ea..a0d80b1 100644
--- a/docs/_sources/models.rst.txt
+++ b/docs/_sources/models.rst.txt
@@ -1,3 +1,5 @@
 Models
 ======
 Coming soon
+
+Until that, consider reading reference on models.
diff --git a/docs/_sources/modules.rst.txt b/docs/_sources/modules.rst.txt
index 9d4ee42..601de40 100644
--- a/docs/_sources/modules.rst.txt
+++ b/docs/_sources/modules.rst.txt
@@ -4,5 +4,8 @@ XAIB modules reference
 .. toctree::
     xaib/base
     xaib/cases
+    xaib/datasets
     xaib/explainers
+    xaib/metrics
+    xaib/models
     xaib/utils
diff --git a/docs/_sources/results.rst.txt b/docs/_sources/results.rst.txt
index c818ac4..32265ec 100644
--- a/docs/_sources/results.rst.txt
+++ b/docs/_sources/results.rst.txt
@@ -1,2 +1,10 @@
 Evaluation results
 ==================
+
+Evaluation results are available for each explanation type separately
+
+.. toctree::
+    :maxdepth: 1
+
+    results/feature_importance
+    results/example_selection
\ No newline at end of file
diff --git a/docs/_sources/xaib/datasets.rst.txt b/docs/_sources/xaib/datasets.rst.txt
new file mode 100644
index 0000000..32333ba
--- /dev/null
+++ b/docs/_sources/xaib/datasets.rst.txt
@@ -0,0 +1,11 @@
+xaib.datasets
+=============
+
+.. autoclass:: xaib.datasets.synthetic_dataset.SyntheticDataset
+   :members:
+
+|
+
+|
+
+|
diff --git a/docs/_sources/xaib/metrics.example_selection.rst.txt b/docs/_sources/xaib/metrics.example_selection.rst.txt
new file mode 100644
index 0000000..9d98ac6
--- /dev/null
+++ b/docs/_sources/xaib/metrics.example_selection.rst.txt
@@ -0,0 +1,38 @@
+xaib.metrics.example_selection
+==============================
+
+.. autoclass:: xaib.metrics.example_selection.covariate_regularity.CovariateRegularity
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.example_selection.parameter_randomization_check.ParameterRandomizationCheck
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.example_selection.small_noise_check.SmallNoiseCheck
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.example_selection.target_discriminativeness.TargetDiscriminativeness
+   :members:
+
+|
+
+|
+
+|
diff --git a/docs/_sources/xaib/metrics.feature_importance.rst.txt b/docs/_sources/xaib/metrics.feature_importance.rst.txt
new file mode 100644
index 0000000..348597c
--- /dev/null
+++ b/docs/_sources/xaib/metrics.feature_importance.rst.txt
@@ -0,0 +1,56 @@
+xaib.metrics.feature_importance
+===============================
+
+.. autoclass:: xaib.metrics.feature_importance.covariate_regularity.CovariateRegularity
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.label_difference.LabelDifference
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.other_disagreement.OtherDisagreement
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.parameter_randomization_check.ParameterRandomizationCheck
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.small_noise_check.SmallNoiseCheck
+   :members:
+
+|
+
+|
+
+|
+
+.. autoclass:: xaib.metrics.feature_importance.sparsity.Sparsity
+   :members:
+
+|
+
+|
+
+|
\ No newline at end of file
diff --git a/docs/_sources/xaib/metrics.rst.txt b/docs/_sources/xaib/metrics.rst.txt
new file mode 100644
index 0000000..e8fc312
--- /dev/null
+++ b/docs/_sources/xaib/metrics.rst.txt
@@ -0,0 +1,6 @@
+xaib.metrics
+============
+
+.. toctree::
+    metrics.feature_importance
+    metrics.example_selection
diff --git a/docs/_sources/xaib/models.rst.txt b/docs/_sources/xaib/models.rst.txt
new file mode 100644
index 0000000..805af61
--- /dev/null
+++ b/docs/_sources/xaib/models.rst.txt
@@ -0,0 +1,2 @@
+xaib.models
+===========
diff --git a/docs/cases.html b/docs/cases.html
index f30f307..4c925a7 100644
--- a/docs/cases.html
+++ b/docs/cases.html
@@ -1 +1 @@
-                        Cases — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Cases
 
Coming soon
 
 
 
 
 
 
   
\ No newline at end of file
+                        Cases — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Cases
 
Coming soon
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/datasets.html b/docs/datasets.html
index 174a367..3ecab76 100644
--- a/docs/datasets.html
+++ b/docs/datasets.html
@@ -1 +1 @@
-                        Datasets — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Datasets
 
Coming soon
 
 
 
 
 
 
   
\ No newline at end of file
+                        Datasets — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Datasets
 
Coming soon
 
Until that, consider reading reference on datasets.
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/genindex.html b/docs/genindex.html
index 4cc3064..41b317a 100644
--- a/docs/genindex.html
+++ b/docs/genindex.html
@@ -1 +1 @@
-                        Index — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Index
 
 _ | A | B | C | D | E | K | L | M | P | R | S 
 
_
 
  
 
A
 
  
 
B
 
   
 
C
 
   
 
D
 
  
 
E
 
   
 
K
 
  
 
L
 
   
 
M
 
   
 
P
 
  
 
R
 
  
 
S
 
   
 
 
 
 
 
   
\ No newline at end of file
+                        Index — xai-benchmark 0.2.0-alpha documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Index
 
 _ | A | B | C | D | E | K | L | M | O | P | R | S | T 
 
_
 
  
 
A
 
  
 
B
 
   
 
C
 
   
 
D
 
  
 
E
 
   
 
K
 
  
 
L
 
   
 
M
 
   
 
O
 
  
 
P
 
  
 
R
 
  
 
S
 
   
 
T
 
  
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/metrics.html b/docs/metrics.html
index e7bacec..5bc6dbd 100644
--- a/docs/metrics.html
+++ b/docs/metrics.html
@@ -1 +1 @@
-                        Metrics — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Metrics
 
Coming soon
 
 
 
 
 
 
   
\ No newline at end of file
+                        Metrics — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Metrics
 
Coming soon
 
Until that, consider reading reference on metrics.
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/models.html b/docs/models.html
index e2631bd..bc0a634 100644
--- a/docs/models.html
+++ b/docs/models.html
@@ -1 +1 @@
-                        Models — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Models
 
Coming soon
 
 
 
 
 
 
   
\ No newline at end of file
+                        Models — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Models
 
Coming soon
 
Until that, consider reading reference on models.
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/modules.html b/docs/modules.html
index 6000ee2..79377d1 100644
--- a/docs/modules.html
+++ b/docs/modules.html
@@ -1 +1 @@
-                        XAIB modules reference — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
XAIB modules reference
 
  
 
 
 
 
 
 
   
\ No newline at end of file
+                        XAIB modules reference — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
XAIB modules reference
 
  
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/objects.inv b/docs/objects.inv
index 8bc8f33..816eb05 100644
Binary files a/docs/objects.inv and b/docs/objects.inv differ
diff --git a/docs/results.html b/docs/results.html
index 07c734b..77b1053 100644
--- a/docs/results.html
+++ b/docs/results.html
@@ -1,375 +1 @@
-
-
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
-  
-   
-   
-   
-    Evaluation results — xai-benchmark 0.2.0-alpha documentation
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-    
-     
-      
-       
-       
-      
-     
-    
-    
-    
-    
-    
-     Skip to content
-    
-    

-     
-    

-    

-     
-     

-      

-       

-        

-         

-          
-         

-        

-       

-       

-        

-         

-          
-         

-        

-       

-       

-        

-         

-          

-           Evaluation results
-           
-            ¶
-           
-          

-         

-        

-       

-      

-     

-    

-    
-     
-      
-     
-     
-      

-       
-       
-       

-       

-       
-      

-     
-    
-    
-     
-      
-     
-     
-      

-       
-       
-       

-       

-       
-      

-     
-    
-    
-    
-    
-   
-  
- 
-
\ No newline at end of file
+                        Evaluation results — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
diff --git a/docs/searchindex.js b/docs/searchindex.js
index 25fca54..1e8c543 100644
--- a/docs/searchindex.js
+++ b/docs/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["cases", "contribute", "contribute/add_dataset", "contribute/add_explainer", "contribute/add_metric", "contribute/add_model", "datasets", "index", "installation", "metrics", "models", "modules", "results", "use_cases", "use_cases/evaluate_method", "use_cases/try_method", "xaib/base", "xaib/cases", "xaib/cases.example_selection", "xaib/cases.feature_importance", "xaib/explainers", "xaib/explainers.example_selection", "xaib/explainers.feature_importance", "xaib/utils"], "filenames": ["cases.rst", "contribute.rst", "contribute/add_dataset.rst", "contribute/add_explainer.rst", "contribute/add_metric.rst", "contribute/add_model.rst", "datasets.rst", "index.rst", "installation.rst", "metrics.rst", "models.rst", "modules.rst", "results.rst", "use_cases.rst", "use_cases/evaluate_method.rst", "use_cases/try_method.rst", "xaib/base.rst", "xaib/cases.rst", "xaib/cases.example_selection.rst", "xaib/cases.feature_importance.rst", "xaib/explainers.rst", "xaib/explainers.example_selection.rst", "xaib/explainers.feature_importance.rst", "xaib/utils.rst"], "titles": ["Cases", "How to contribute", "Add dataset", "Add explainer", "Add metric", "Add model", "Datasets", "Welcome to XAIB!", "Installation", "Metrics", "Models", "XAIB modules reference", "Evaluation results", "Use cases", "Evaluate a method", "Try a method", "xaib.base", "xaib.cases", "xaib.cases.example_selection", "xaib.cases.feature_importance", "xaib.explainers", "xaib.explainers.example_selection", "xaib.explainers.feature_importance", "xaib.utils"], "terms": {"come": [0, 6, 9, 10], "soon": [0, 6, 9, 10], "ani": [1, 2, 4, 16, 18, 19, 21, 22], "ar": [1, 3, 4, 5], "welcom": 1, "you": [1, 2, 3, 4, 5, 8, 14, 15], "can": [1, 2, 3, 5, 8, 15, 16, 18, 19, 21, 22], "help": 1, "extend": [1, 2], "pictur": 1, "xai": [1, 7, 8], "method": [1, 5, 7, 13, 16, 21, 22], "qualiti": [1, 4], "ad": [1, 2, 3, 4, 5], "add": 1, "dataset": [1, 4, 5, 7, 11, 15, 16, 18, 19], "model": [1, 2, 3, 4, 7, 11, 14, 15, 16, 18, 19, 21, 22], "explain": [1, 2, 4, 5, 8, 11, 14, 15, 16], "metric": [1, 7, 16], "If": [1, 15], "want": [1, 15], "benchmark": [1, 3, 7, 8], "new": 1, "case": [1, 4, 5, 7, 11, 14, 16], "pleas": 1, "fill": [1, 2, 5], "issu": 1, "mai": [2, 16, 18, 19, 21, 22], "our": 2, "understand": 2, "how": [2, 3, 4, 5, 7], "differ": [2, 5, 7], "behav": 2, "context": 2, "domain": 2, "task": 2, "To": [2, 3, 5, 8, 14], "your": [2, 3, 4, 5], "should": [2, 4, 5, 8, 14, 16, 18, 19, 21, 22], "provid": [2, 5, 7], "which": [2, 3, 4, 8, 15, 16], "download": 2, "access": 2, "prepar": 2, "from": [2, 3, 4, 5, 14, 15], "disk": 2, "first": [2, 4, 5], "need": [2, 3, 4, 5, 8, 16, 18, 19, 21, 22], "requir": [2, 5], "interfac": [2, 15], "field": 2, "import": [2, 3, 4, 5, 8, 14, 15], "numpi": [2, 3, 5], "np": [2, 3, 4, 5], "xaib": [2, 3, 4, 5, 8, 14, 15], "class": [2, 3, 4, 5, 16, 18, 19, 21, 22, 23], "newdataset": 2, "here": [2, 4, 5], "document": [2, 5, 16, 18, 19, 21, 22], "def": [2, 3, 4, 5], "__init__": [2, 3, 4, 5, 11, 16, 17, 18, 19, 20, 21, 22, 23], "self": [2, 3, 4, 5, 16, 21, 22], "split": 2, "arg": [2, 3, 4, 5, 16, 18, 19, 22], "kwarg": [2, 3, 4, 5, 16, 18, 19, 21, 22], "none": [2, 3, 4, 5, 16, 18, 19, 21, 22, 23], "super": [2, 3, 4, 5], "It": [2, 5], "i": [2, 3, 4, 5, 7, 15, 16, 18, 19, 21, 22], "set": [2, 5, 7], "name": [2, 3, 4, 5, 14, 16], "us": [2, 3, 4, 5, 7, 8, 15], "identifi": [2, 5], "new_dataset": 2, "while": 2, "cach": 2, "defin": [2, 4], "etc": 2, "train": [2, 5, 15, 16, 18, 19, 21, 22], "_data": 2, "0": [2, 3, 4, 5, 15, 22], "1": [2, 4, 5, 15, 22, 23], "2": 2, "3": 2, "4": 2, "5": 2, "6": 2, "7": 2, "8": 2, "_label": 2, "elif": 2, "9": 2, "10": [2, 3, 5, 14, 15], "11": 2, "12": 2, "13": 2, "14": 2, "__getitem__": 2, "index": 2, "thi": [2, 3, 15], "form": [2, 16], "return": [2, 3, 4, 5, 16], "item": [2, 4, 15, 16], "dict": [2, 4, 16], "str": [2, 5, 16], "ndarrai": 2, "asarrai": 2, "label": [2, 4, 5, 15, 22], "__len__": 2, "len": [2, 3, 5], "befor": [2, 3, 4, 5], "implement": [2, 3, 4, 5, 16], "directli": [2, 3, 4, 5, 8], "sourc": [2, 3, 4, 5, 16, 18, 19, 21, 22, 23], "code": [2, 3, 4, 5], "would": [2, 3, 4, 5], "work": [2, 3, 4, 5], "standard": [2, 3, 4, 5], "setup": [2, 3, 4, 5, 14], "evalu": [2, 3, 4, 5, 7, 11, 13, 15, 16], "datasetfactori": [2, 3, 4, 5, 14, 15], "modelfactori": [2, 3, 4, 5, 14, 15], "feature_import": [2, 3, 4, 5, 11, 15, 17, 20], "explainerfactori": [2, 3, 4, 5, 14, 15], "experimentfactori": [2, 3, 5, 14], "util": [2, 3, 4, 5, 11, 14], "visualize_result": [2, 3, 4, 5, 14], "train_d": [2, 3, 4, 5, 14, 15, 21, 22], "test_d": [2, 3, 4, 5, 14, 15], "And": 2, "pass": [2, 5, 16, 18, 19, 21, 22], "further": 2, "get": [2, 3, 4, 5, 14, 15], "svm": [2, 3, 4, 15], "all": [2, 3, 4, 5, 8, 14, 15, 16, 18, 19, 21, 22], "experiment_factori": [2, 3, 5, 14], "repo_path": [2, 3, 5, 14], "result": [2, 3, 4, 5, 7, 14, 15], "batch_siz": [2, 3, 4, 5, 14, 23], "experi": [2, 3, 4, 5, 14], "png": [2, 3, 4, 5, 14], "final": [2, 3, 5], "do": [2, 3, 5, 15], "modul": [2, 3, 5, 7], "make": [2, 3, 5, 16, 21, 22], "constructor": [2, 3, 5], "factori": [2, 3, 5], "dataset_factori": 2, "py": [2, 3, 4, 5], "function": [2, 3, 5], "build": [2, 3, 5], "default": [2, 14, 15, 16, 18, 19, 21, 22], "_constructor": [2, 3, 5], "lambda": [2, 3, 5], "heart": 3, "thei": [3, 4], "being": 3, "thorougli": 3, "more": [3, 4], "them": [3, 15, 16, 18, 19, 21, 22], "we": 3, "know": 3, "less": [3, 4], "demand": 3, "than": 3, "": [3, 16, 18, 19, 21, 22], "easier": 3, "newexplain": 3, "essenti": 3, "put": [3, 5, 16], "tabl": 3, "correctli": [3, 4], "new_explain": 3, "predict": [3, 4, 5, 11, 15, 16, 20, 21, 22], "x": [3, 5, 16, 21, 22, 23], "random": [3, 4], "rand": [3, 4], "synthet": [3, 4, 5, 14, 15], "explainer_factori": 3, "wai": [4, 7], "numer": 4, "assess": 4, "part": 4, "object": 4, "accept": [4, 16], "data": [4, 16, 18, 19, 21, 22, 23], "some": [4, 5, 14, 16], "valu": [4, 16], "direct": 4, "order": 4, "displai": 4, "short": 4, "what": 4, "measur": [4, 7], "denot": 4, "consid": 4, "better": 4, "greater": 4, "newmetr": 4, "d": [4, 18, 19], "new_metr": 4, "down": 4, "comput": 4, "expl_kwarg": 4, "dl": 4, "simpledataload": [4, 11, 23], "_d": 4, "batch": [4, 15], "tqdm": 4, "explan": [4, 15, 16], "ex": 4, "expl": [4, 16], "_model": 4, "sinc": 4, "low": 4, "level": [4, 16], "special": [4, 16], "treatment": 4, "when": 4, "basic": 4, "append": 4, "exist": [4, 14], "choic": 4, "metrics_kwarg": [4, 16], "other_disagr": 4, "list": [4, 23], "coher": 4, "coherencecas": [4, 11, 17, 19], "add_metr": [4, 11, 16], "coherence_cas": 4, "_metric_obj": 4, "inform": 5, "good": 5, "particular": 5, "ha": 5, "mani": 5, "For": 5, "exampl": [5, 21], "fit": 5, "abl": [5, 16, 18, 19, 21, 22], "see": [5, 15], "specif": 5, "base": [5, 11], "model_factori": 5, "newmodel": 5, "const": 5, "new_model": 5, "y": [5, 16, 23], "arrai": 5, "_": 5, "rang": [5, 15], "save": [5, 14], "filepath": 5, "open": [5, 7], "w": 5, "f": [5, 16], "write": 5, "load": [5, 16, 18, 19, 21, 22], "r": [5, 8], "float": 5, "read": 5, "doe": 5, "anyth": 5, "just": [5, 16], "intern": 5, "state": 5, "shap": 5, "an": [7, 15, 16], "compar": [7, 15], "broad": 7, "aim": 7, "aspect": 7, "interpret": 7, "instal": 7, "contribut": 7, "refer": 7, "latest": 8, "version": 8, "packag": 8, "run": [8, 14, 15], "pip3": 8, "rememb": 8, "creat": [8, 14, 16, 18, 19, 21, 22], "virtual": 8, "environ": 8, "one": [8, 15], "after": 8, "verifi": 8, "print": [8, 15], "out": 8, "its": 8, "__version__": 8, "also": [8, 15, 16], "explainers_requir": 8, "txt": 8, "done": 8, "http": 8, "raw": 8, "githubusercont": 8, "com": 8, "oxid15": 8, "master": 8, "compactnesscas": [11, 17, 19], "continuitycas": [11, 17, 18, 19], "contrastivitycas": [11, 17, 19], "correctnesscas": [11, 17, 19], "covariatecomplexitycas": [11, 17, 19], "example_select": [11, 14, 17, 20], "constantexplain": [11, 20, 21, 22], "limeexplain": [11, 15, 20, 22], "linearregressionexplain": [11, 20, 22], "randomexplain": [11, 20, 21, 22], "shapexplain": [11, 20, 22], "knnexplain": [11, 20, 21], "batch_rms": [11, 23], "entropi": [11, 23], "batch_gini": [11, 23], "batch_count_eq": [11, 23], "minmax_norm": [11, 23], "try": 13, "knn": 14, "chosen": 14, "plot": 14, "folder": 14, "have": [15, 16, 18, 19, 21, 22], "same": 15, "input": [15, 16], "output": 15, "allow": 15, "easili": 15, "lime_explain": [15, 22], "obtain": 15, "sampl": 15, "wrapper": 16, "around": 16, "collect": 16, "ml": 16, "infer": [16, 21, 22], "other": 16, "solut": 16, "given": 16, "certain": 16, "respons": 16, "call": [16, 18, 19, 21, 22], "successor": [16, 18, 19, 21, 22], "initi": [16, 18, 19, 21, 22], "meta": [16, 18, 19, 21, 22], "argument": [16, 18, 19, 21, 22], "relat": [16, 18, 19, 21, 22], "hyperparamet": [16, 18, 19, 21, 22], "architectur": [16, 18, 19, 21, 22], "addit": [16, 18, 19, 21, 22], "paramet": [16, 18, 19, 21, 22], "superclass": [16, 18, 19, 21, 22], "log": [16, 18, 19, 21, 22], "everyth": [16, 18, 19, 21, 22], "worth": [16, 18, 19, 21, 22], "about": [16, 18, 19, 21, 22], "wa": [16, 18, 19, 21, 22], "either": [16, 18, 19, 21, 22], "param": [16, 18, 19, 21, 22], "meta_prefix": [16, 18, 19, 21, 22], "encapsul": [16, 21, 22], "includ": [16, 21, 22], "preprocess": [16, 21, 22], "step": [16, 21, 22], "suffici": [16, 21, 22], "kind": 16, "e": 16, "g": 16, "anoth": 16, "repres": 16, "high": 16, "properti": 16, "against": 16, "popul": 16, "noisy_d": [18, 19], "noisy_model": 19, "constant_explain": [21, 22], "knn_explain": 21, "random_explain": [21, 22], "constant": 22, "linear_regression_explain": 22, "get_coef": 22, "shift": 22, "magnitud": 22, "shap_explain": 22, "bx": 23, "bool": 23, "int": 23}, "objects": {"xaib.base": [[16, 0, 1, "", "Case"], [16, 0, 1, "", "Dataset"], [16, 0, 1, "", "Explainer"], [16, 0, 1, "", "Model"]], "xaib.base.Case": [[16, 1, 1, "", "__init__"], [16, 1, 1, "", "add_metric"], [16, 1, 1, "", "evaluate"]], "xaib.base.Dataset": [[16, 1, 1, "", "__init__"]], "xaib.base.Explainer": [[16, 1, 1, "", "__init__"], [16, 1, 1, "", "predict"]], "xaib.base.Model": [[16, 1, 1, "", "__init__"], [16, 1, 1, "", "predict"]], "xaib.cases.example_selection": [[18, 0, 1, "", "ContinuityCase"]], "xaib.cases.example_selection.ContinuityCase": [[18, 1, 1, "", "__init__"]], "xaib.cases.feature_importance": [[19, 0, 1, "", "CoherenceCase"], [19, 0, 1, "", "CompactnessCase"], [19, 0, 1, "", "ContinuityCase"], [19, 0, 1, "", "ContrastivityCase"], [19, 0, 1, "", "CorrectnessCase"], [19, 0, 1, "", "CovariateComplexityCase"]], "xaib.cases.feature_importance.CoherenceCase": [[19, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.CompactnessCase": [[19, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.ContinuityCase": [[19, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.ContrastivityCase": [[19, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.CorrectnessCase": [[19, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.CovariateComplexityCase": [[19, 1, 1, "", "__init__"]], "xaib.explainers.example_selection.constant_explainer": [[21, 0, 1, "", "ConstantExplainer"]], "xaib.explainers.example_selection.constant_explainer.ConstantExplainer": [[21, 1, 1, "", "__init__"], [21, 1, 1, "", "predict"]], "xaib.explainers.example_selection.knn_explainer": [[21, 0, 1, "", "KNNExplainer"]], "xaib.explainers.example_selection.knn_explainer.KNNExplainer": [[21, 1, 1, "", "__init__"], [21, 1, 1, "", "predict"]], "xaib.explainers.example_selection.random_explainer": [[21, 0, 1, "", "RandomExplainer"]], "xaib.explainers.example_selection.random_explainer.RandomExplainer": [[21, 1, 1, "", "__init__"], [21, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.constant_explainer": [[22, 0, 1, "", "ConstantExplainer"]], "xaib.explainers.feature_importance.constant_explainer.ConstantExplainer": [[22, 1, 1, "", "__init__"], [22, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.lime_explainer": [[22, 0, 1, "", "LimeExplainer"]], "xaib.explainers.feature_importance.lime_explainer.LimeExplainer": [[22, 1, 1, "", "__init__"], [22, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.linear_regression_explainer": [[22, 0, 1, "", "LinearRegressionExplainer"]], "xaib.explainers.feature_importance.linear_regression_explainer.LinearRegressionExplainer": [[22, 1, 1, "", "__init__"], [22, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.random_explainer": [[22, 0, 1, "", "RandomExplainer"]], "xaib.explainers.feature_importance.random_explainer.RandomExplainer": [[22, 1, 1, "", "__init__"], [22, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.shap_explainer": [[22, 0, 1, "", "ShapExplainer"]], "xaib.explainers.feature_importance.shap_explainer.ShapExplainer": [[22, 1, 1, "", "__init__"], [22, 1, 1, "", "predict"]], "xaib.utils": [[23, 0, 1, "", "SimpleDataloader"], [23, 2, 1, "", "batch_count_eq"], [23, 2, 1, "", "batch_gini"], [23, 2, 1, "", "batch_rmse"], [23, 2, 1, "", "entropy"], [23, 2, 1, "", "minmax_normalize"]], "xaib.utils.SimpleDataloader": [[23, 1, 1, "", "__init__"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "function", "Python function"]}, "titleterms": {"case": [0, 13, 17, 18, 19], "how": 1, "contribut": 1, "add": [2, 3, 4, 5], "dataset": [2, 6], "creat": [2, 3, 4, 5], "data": 2, "wrapper": [2, 3, 5], "test": [2, 3, 4, 5], "new": [2, 3, 4, 5], "integr": [2, 3, 4, 5], "explain": [3, 20, 21, 22], "metric": [4, 9], "model": [5, 10], "welcom": 7, "xaib": [7, 11, 16, 17, 18, 19, 20, 21, 22, 23], "instal": 8, "modul": 11, "refer": 11, "evalu": [12, 14], "result": 12, "us": 13, "method": [14, 15], "try": 15, "base": 16, "example_select": [18, 21], "feature_import": [19, 22], "util": 23}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.todo": 2, "sphinx.ext.viewcode": 1, "sphinx": 57}, "alltitles": {"Cases": [[0, "cases"]], "How to contribute": [[1, "how-to-contribute"]], "Add dataset": [[2, "add-dataset"]], "Create data wrapper": [[2, "create-data-wrapper"]], "Test new dataset": [[2, "test-new-dataset"]], "Integrate new dataset": [[2, "integrate-new-dataset"]], "Add explainer": [[3, "add-explainer"]], "Create explainer wrapper": [[3, "create-explainer-wrapper"]], "Test new explainer": [[3, "test-new-explainer"]], "Integrate new explainer": [[3, "integrate-new-explainer"]], "Add metric": [[4, "add-metric"]], "Create metric": [[4, "create-metric"]], "Test new metric": [[4, "test-new-metric"]], "Integrate new metric": [[4, "integrate-new-metric"]], "Add model": [[5, "add-model"]], "Create model wrapper": [[5, "create-model-wrapper"]], "Test new model": [[5, "test-new-model"]], "Integrate new model": [[5, "integrate-new-model"]], "Datasets": [[6, "datasets"]], "Welcome to XAIB!": [[7, "welcome-to-xaib"]], "Installation": [[8, "installation"]], "Metrics": [[9, "metrics"]], "Models": [[10, "models"]], "XAIB modules reference": [[11, "xaib-modules-reference"]], "Evaluation results": [[12, "evaluation-results"]], "Use cases": [[13, "use-cases"]], "Evaluate a method": [[14, "evaluate-a-method"]], "Try a method": [[15, "try-a-method"]], "xaib.base": [[16, "xaib-base"]], "xaib.cases": [[17, "xaib-cases"]], "xaib.cases.example_selection": [[18, "xaib-cases-example-selection"]], "xaib.cases.feature_importance": [[19, "xaib-cases-feature-importance"]], "xaib.explainers": [[20, "xaib-explainers"]], "xaib.explainers.example_selection": [[21, "xaib-explainers-example-selection"]], "xaib.explainers.feature_importance": [[22, "xaib-explainers-feature-importance"]], "xaib.utils": [[23, "xaib-utils"]]}, "indexentries": {"case (class in xaib.base)": [[16, "xaib.base.Case"]], "dataset (class in xaib.base)": [[16, "xaib.base.Dataset"]], "explainer (class in xaib.base)": [[16, "xaib.base.Explainer"]], "model (class in xaib.base)": [[16, "xaib.base.Model"]], "__init__() (xaib.base.case method)": [[16, "xaib.base.Case.__init__"]], "__init__() (xaib.base.dataset method)": [[16, "xaib.base.Dataset.__init__"]], "__init__() (xaib.base.explainer method)": [[16, "xaib.base.Explainer.__init__"]], "__init__() (xaib.base.model method)": [[16, "xaib.base.Model.__init__"]], "add_metric() (xaib.base.case method)": [[16, "xaib.base.Case.add_metric"]], "evaluate() (xaib.base.case method)": [[16, "xaib.base.Case.evaluate"]], "predict() (xaib.base.explainer method)": [[16, "xaib.base.Explainer.predict"]], "predict() (xaib.base.model method)": [[16, "xaib.base.Model.predict"]], "continuitycase (class in xaib.cases.example_selection)": [[18, "xaib.cases.example_selection.ContinuityCase"]], "__init__() (xaib.cases.example_selection.continuitycase method)": [[18, "xaib.cases.example_selection.ContinuityCase.__init__"]], "coherencecase (class in xaib.cases.feature_importance)": [[19, "xaib.cases.feature_importance.CoherenceCase"]], "compactnesscase (class in xaib.cases.feature_importance)": [[19, "xaib.cases.feature_importance.CompactnessCase"]], "continuitycase (class in xaib.cases.feature_importance)": [[19, "xaib.cases.feature_importance.ContinuityCase"]], "contrastivitycase (class in xaib.cases.feature_importance)": [[19, "xaib.cases.feature_importance.ContrastivityCase"]], "correctnesscase (class in xaib.cases.feature_importance)": [[19, "xaib.cases.feature_importance.CorrectnessCase"]], "covariatecomplexitycase (class in xaib.cases.feature_importance)": [[19, "xaib.cases.feature_importance.CovariateComplexityCase"]], "__init__() (xaib.cases.feature_importance.coherencecase method)": [[19, "xaib.cases.feature_importance.CoherenceCase.__init__"]], "__init__() (xaib.cases.feature_importance.compactnesscase method)": [[19, "xaib.cases.feature_importance.CompactnessCase.__init__"]], "__init__() (xaib.cases.feature_importance.continuitycase method)": [[19, "xaib.cases.feature_importance.ContinuityCase.__init__"]], "__init__() (xaib.cases.feature_importance.contrastivitycase method)": [[19, "xaib.cases.feature_importance.ContrastivityCase.__init__"]], "__init__() (xaib.cases.feature_importance.correctnesscase method)": [[19, "xaib.cases.feature_importance.CorrectnessCase.__init__"]], "__init__() (xaib.cases.feature_importance.covariatecomplexitycase method)": [[19, "xaib.cases.feature_importance.CovariateComplexityCase.__init__"]], "constantexplainer (class in xaib.explainers.example_selection.constant_explainer)": [[21, "xaib.explainers.example_selection.constant_explainer.ConstantExplainer"]], "knnexplainer (class in xaib.explainers.example_selection.knn_explainer)": [[21, "xaib.explainers.example_selection.knn_explainer.KNNExplainer"]], "randomexplainer (class in xaib.explainers.example_selection.random_explainer)": [[21, "xaib.explainers.example_selection.random_explainer.RandomExplainer"]], "__init__() (xaib.explainers.example_selection.constant_explainer.constantexplainer method)": [[21, "xaib.explainers.example_selection.constant_explainer.ConstantExplainer.__init__"]], "__init__() (xaib.explainers.example_selection.knn_explainer.knnexplainer method)": [[21, "xaib.explainers.example_selection.knn_explainer.KNNExplainer.__init__"]], "__init__() (xaib.explainers.example_selection.random_explainer.randomexplainer method)": [[21, "xaib.explainers.example_selection.random_explainer.RandomExplainer.__init__"]], "predict() (xaib.explainers.example_selection.constant_explainer.constantexplainer method)": [[21, "xaib.explainers.example_selection.constant_explainer.ConstantExplainer.predict"]], "predict() (xaib.explainers.example_selection.knn_explainer.knnexplainer method)": [[21, "xaib.explainers.example_selection.knn_explainer.KNNExplainer.predict"]], "predict() (xaib.explainers.example_selection.random_explainer.randomexplainer method)": [[21, "xaib.explainers.example_selection.random_explainer.RandomExplainer.predict"]], "constantexplainer (class in xaib.explainers.feature_importance.constant_explainer)": [[22, "xaib.explainers.feature_importance.constant_explainer.ConstantExplainer"]], "limeexplainer (class in xaib.explainers.feature_importance.lime_explainer)": [[22, "xaib.explainers.feature_importance.lime_explainer.LimeExplainer"]], "linearregressionexplainer (class in xaib.explainers.feature_importance.linear_regression_explainer)": [[22, "xaib.explainers.feature_importance.linear_regression_explainer.LinearRegressionExplainer"]], "randomexplainer (class in xaib.explainers.feature_importance.random_explainer)": [[22, "xaib.explainers.feature_importance.random_explainer.RandomExplainer"]], "shapexplainer (class in xaib.explainers.feature_importance.shap_explainer)": [[22, "xaib.explainers.feature_importance.shap_explainer.ShapExplainer"]], "__init__() (xaib.explainers.feature_importance.constant_explainer.constantexplainer method)": [[22, "xaib.explainers.feature_importance.constant_explainer.ConstantExplainer.__init__"]], "__init__() (xaib.explainers.feature_importance.lime_explainer.limeexplainer method)": [[22, "xaib.explainers.feature_importance.lime_explainer.LimeExplainer.__init__"]], "__init__() (xaib.explainers.feature_importance.linear_regression_explainer.linearregressionexplainer method)": [[22, "xaib.explainers.feature_importance.linear_regression_explainer.LinearRegressionExplainer.__init__"]], "__init__() (xaib.explainers.feature_importance.random_explainer.randomexplainer method)": [[22, "xaib.explainers.feature_importance.random_explainer.RandomExplainer.__init__"]], "__init__() (xaib.explainers.feature_importance.shap_explainer.shapexplainer method)": [[22, "xaib.explainers.feature_importance.shap_explainer.ShapExplainer.__init__"]], "predict() (xaib.explainers.feature_importance.constant_explainer.constantexplainer method)": [[22, "xaib.explainers.feature_importance.constant_explainer.ConstantExplainer.predict"]], "predict() (xaib.explainers.feature_importance.lime_explainer.limeexplainer method)": [[22, "xaib.explainers.feature_importance.lime_explainer.LimeExplainer.predict"]], "predict() (xaib.explainers.feature_importance.linear_regression_explainer.linearregressionexplainer method)": [[22, "xaib.explainers.feature_importance.linear_regression_explainer.LinearRegressionExplainer.predict"]], "predict() (xaib.explainers.feature_importance.random_explainer.randomexplainer method)": [[22, "xaib.explainers.feature_importance.random_explainer.RandomExplainer.predict"]], "predict() (xaib.explainers.feature_importance.shap_explainer.shapexplainer method)": [[22, "xaib.explainers.feature_importance.shap_explainer.ShapExplainer.predict"]], "simpledataloader (class in xaib.utils)": [[23, "xaib.utils.SimpleDataloader"]], "__init__() (xaib.utils.simpledataloader method)": [[23, "xaib.utils.SimpleDataloader.__init__"]], "batch_count_eq() (in module xaib.utils)": [[23, "xaib.utils.batch_count_eq"]], "batch_gini() (in module xaib.utils)": [[23, "xaib.utils.batch_gini"]], "batch_rmse() (in module xaib.utils)": [[23, "xaib.utils.batch_rmse"]], "entropy() (in module xaib.utils)": [[23, "xaib.utils.entropy"]], "minmax_normalize() (in module xaib.utils)": [[23, "xaib.utils.minmax_normalize"]]}})
\ No newline at end of file
+Search.setIndex({"docnames": ["cases", "contribute", "contribute/add_dataset", "contribute/add_explainer", "contribute/add_metric", "contribute/add_model", "datasets", "index", "installation", "metrics", "models", "modules", "results", "results/example_selection", "results/feature_importance", "use_cases", "use_cases/evaluate_method", "use_cases/try_method", "xaib/base", "xaib/cases", "xaib/cases.example_selection", "xaib/cases.feature_importance", "xaib/datasets", "xaib/explainers", "xaib/explainers.example_selection", "xaib/explainers.feature_importance", "xaib/metrics", "xaib/metrics.example_selection", "xaib/metrics.feature_importance", "xaib/models", "xaib/utils"], "filenames": ["cases.rst", "contribute.rst", "contribute/add_dataset.rst", "contribute/add_explainer.rst", "contribute/add_metric.rst", "contribute/add_model.rst", "datasets.rst", "index.rst", "installation.rst", "metrics.rst", "models.rst", "modules.rst", "results.rst", "results/example_selection.rst", "results/feature_importance.rst", "use_cases.rst", "use_cases/evaluate_method.rst", "use_cases/try_method.rst", "xaib/base.rst", "xaib/cases.rst", "xaib/cases.example_selection.rst", "xaib/cases.feature_importance.rst", "xaib/datasets.rst", "xaib/explainers.rst", "xaib/explainers.example_selection.rst", "xaib/explainers.feature_importance.rst", "xaib/metrics.rst", "xaib/metrics.example_selection.rst", "xaib/metrics.feature_importance.rst", "xaib/models.rst", "xaib/utils.rst"], "titles": ["Cases", "How to contribute", "Add dataset", "Add explainer", "Add metric", "Add model", "Datasets", "Welcome to XAIB!", "Installation", "Metrics", "Models", "XAIB modules reference", "Evaluation results", "Example Selection Results", "Feature Importance Results", "Use cases", "Evaluate a method", "Try a method", "xaib.base", "xaib.cases", "xaib.cases.example_selection", "xaib.cases.feature_importance", "xaib.datasets", "xaib.explainers", "xaib.explainers.example_selection", "xaib.explainers.feature_importance", "xaib.metrics", "xaib.metrics.example_selection", "xaib.metrics.feature_importance", "xaib.models", "xaib.utils"], "terms": {"come": [0, 6, 9, 10], "soon": [0, 6, 9, 10], "ani": [1, 2, 4, 18, 20, 21, 24, 25, 27, 28], "ar": [1, 3, 4, 5, 12, 27, 28], "welcom": 1, "you": [1, 2, 3, 4, 5, 8, 16, 17], "can": [1, 2, 3, 5, 8, 17, 18, 20, 21, 24, 25, 27, 28], "help": 1, "extend": [1, 2], "pictur": 1, "xai": [1, 7, 8], "method": [1, 5, 7, 15, 18, 24, 25, 27, 28], "qualiti": [1, 4, 27], "ad": [1, 2, 3, 4, 5], "add": [1, 27], "dataset": [1, 4, 5, 7, 11, 17, 18, 20, 21, 27, 28], "model": [1, 2, 3, 4, 7, 11, 16, 17, 18, 20, 21, 24, 25, 27, 28], "explain": [1, 2, 4, 5, 8, 11, 16, 17, 18, 27, 28], "metric": [1, 7, 11, 18], "If": [1, 17], "want": [1, 17], "benchmark": [1, 3, 7, 8], "new": 1, "case": [1, 4, 5, 7, 11, 16, 18, 27, 28], "pleas": [1, 13, 14], "fill": [1, 2, 5], "issu": 1, "mai": [2, 18, 20, 21, 24, 25, 27, 28], "our": 2, "understand": 2, "how": [2, 3, 4, 5, 7, 27, 28], "differ": [2, 5, 7, 27, 28], "behav": 2, "context": 2, "domain": 2, "task": 2, "To": [2, 3, 5, 8, 16, 27, 28], "your": [2, 3, 4, 5], "should": [2, 4, 5, 8, 16, 18, 20, 21, 24, 25, 27, 28], "provid": [2, 5, 7, 27], "which": [2, 3, 4, 8, 17, 18, 28], "download": 2, "access": 2, "prepar": 2, "from": [2, 3, 4, 5, 16, 17, 27, 28], "disk": 2, "first": [2, 4, 5], "need": [2, 3, 4, 5, 8, 18, 20, 21, 24, 25, 27, 28], "requir": [2, 5], "interfac": [2, 17], "field": 2, "import": [2, 3, 4, 5, 8, 12, 16, 17, 27, 28], "numpi": [2, 3, 5], "np": [2, 3, 4, 5], "xaib": [2, 3, 4, 5, 8, 16, 17], "class": [2, 3, 4, 5, 18, 20, 21, 22, 24, 25, 27, 28, 30], "newdataset": 2, "here": [2, 4, 5], "document": [2, 5, 18, 20, 21, 24, 25, 27, 28], "def": [2, 3, 4, 5], "__init__": [2, 3, 4, 5, 11, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 30], "self": [2, 3, 4, 5, 18, 24, 25], "split": [2, 22], "arg": [2, 3, 4, 5, 18, 20, 21, 22, 25, 27, 28], "kwarg": [2, 3, 4, 5, 18, 20, 21, 22, 24, 25, 27, 28], "none": [2, 3, 4, 5, 18, 20, 21, 22, 24, 25, 27, 28, 30], "super": [2, 3, 4, 5], "It": [2, 5, 27, 28], "i": [2, 3, 4, 5, 7, 17, 18, 20, 21, 24, 25, 27, 28], "set": [2, 5, 7, 27], "name": [2, 3, 4, 5, 16, 18, 22], "us": [2, 3, 4, 5, 7, 8, 17, 27, 28], "identifi": [2, 5], "new_dataset": 2, "while": 2, "cach": 2, "defin": [2, 4, 27, 28], "etc": 2, "train": [2, 5, 17, 18, 20, 21, 24, 25, 27, 28], "_data": 2, "0": [2, 3, 4, 5, 17, 22, 25], "1": [2, 4, 5, 17, 25, 27, 28, 30], "2": 2, "3": 2, "4": 2, "5": 2, "6": 2, "7": 2, "8": [2, 22], "_label": 2, "elif": 2, "9": 2, "10": [2, 3, 5, 16, 17], "11": 2, "12": 2, "13": 2, "14": 2, "__getitem__": 2, "index": [2, 28], "thi": [2, 3, 17, 27, 28], "form": [2, 18, 27], "return": [2, 3, 4, 5, 18], "item": [2, 4, 17, 18, 27], "dict": [2, 4, 18, 27, 28], "str": [2, 5, 18], "ndarrai": 2, "asarrai": 2, "label": [2, 4, 5, 17, 25, 27], "__len__": 2, "len": [2, 3, 5], "befor": [2, 3, 4, 5], "implement": [2, 3, 4, 5, 18], "directli": [2, 3, 4, 5, 8], "sourc": [2, 3, 4, 5, 18, 20, 21, 22, 24, 25, 27, 28, 30], "code": [2, 3, 4, 5], "would": [2, 3, 4, 5], "work": [2, 3, 4, 5], "standard": [2, 3, 4, 5], "setup": [2, 3, 4, 5, 16], "evalu": [2, 3, 4, 5, 7, 11, 15, 17, 18], "datasetfactori": [2, 3, 4, 5, 16, 17], "modelfactori": [2, 3, 4, 5, 16, 17], "feature_import": [2, 3, 4, 5, 11, 17, 19, 23, 26], "explainerfactori": [2, 3, 4, 5, 16, 17], "experimentfactori": [2, 3, 5, 16], "util": [2, 3, 4, 5, 11, 16], "visualize_result": [2, 3, 4, 5, 16], "train_d": [2, 3, 4, 5, 16, 17, 24, 25], "test_d": [2, 3, 4, 5, 16, 17], "And": 2, "pass": [2, 5, 18, 20, 21, 24, 25, 27, 28], "further": [2, 28], "get": [2, 3, 4, 5, 16, 17], "svm": [2, 3, 4, 17], "all": [2, 3, 4, 5, 8, 16, 17, 18, 20, 21, 24, 25, 27, 28], "experiment_factori": [2, 3, 5, 16], "repo_path": [2, 3, 5, 16], "result": [2, 3, 4, 5, 7, 16, 17, 28], "batch_siz": [2, 3, 4, 5, 16, 27, 28, 30], "experi": [2, 3, 4, 5, 16], "png": [2, 3, 4, 5, 16], "final": [2, 3, 5], "do": [2, 3, 5, 17, 27, 28], "modul": [2, 3, 5, 7], "make": [2, 3, 5, 18, 24, 25], "constructor": [2, 3, 5], "factori": [2, 3, 5], "dataset_factori": 2, "py": [2, 3, 4, 5], "function": [2, 3, 5], "build": [2, 3, 5], "default": [2, 16, 17, 18, 20, 21, 24, 25, 27, 28], "_constructor": [2, 3, 5], "lambda": [2, 3, 5], "heart": 3, "thei": [3, 4, 27, 28], "being": 3, "thorougli": 3, "more": [3, 4, 27, 28], "them": [3, 17, 18, 20, 21, 24, 25, 27, 28], "we": [3, 27, 28], "know": 3, "less": [3, 4, 27, 28], "demand": 3, "than": 3, "": [3, 18, 20, 21, 24, 25, 27, 28], "easier": 3, "newexplain": 3, "essenti": 3, "put": [3, 5, 18], "tabl": 3, "correctli": [3, 4], "new_explain": 3, "predict": [3, 4, 5, 11, 17, 18, 23, 24, 25], "x": [3, 5, 18, 24, 25, 30], "random": [3, 4, 27, 28], "rand": [3, 4], "synthet": [3, 4, 5, 16, 17], "explainer_factori": 3, "wai": [4, 7], "numer": 4, "assess": 4, "part": 4, "object": 4, "accept": [4, 18], "data": [4, 18, 20, 21, 24, 25, 27, 28, 30], "some": [4, 5, 16, 18], "valu": [4, 18, 27, 28], "direct": 4, "order": 4, "displai": 4, "short": 4, "what": 4, "measur": [4, 7, 27, 28], "denot": 4, "consid": [4, 6, 9, 10, 27, 28], "better": [4, 27, 28], "greater": [4, 27, 28], "newmetr": 4, "d": [4, 20, 21, 27, 28], "new_metr": 4, "down": [4, 13, 14], "comput": [4, 11, 26, 27, 28], "expl_kwarg": [4, 27, 28], "dl": 4, "simpledataload": [4, 11, 30], "_d": 4, "batch": [4, 17, 27, 28], "tqdm": 4, "explan": [4, 12, 17, 18, 27, 28], "ex": 4, "expl": [4, 18, 27, 28], "_model": 4, "sinc": [4, 27], "low": 4, "level": [4, 18], "special": [4, 18], "treatment": 4, "when": [4, 27, 28], "basic": 4, "append": 4, "exist": [4, 16], "choic": 4, "metrics_kwarg": [4, 18], "other_disagr": [4, 28], "list": [4, 28, 30], "coher": 4, "coherencecas": [4, 11, 19, 21], "add_metr": [4, 11, 18], "coherence_cas": 4, "_metric_obj": 4, "inform": [5, 27], "good": 5, "particular": [5, 28], "ha": [5, 27, 28], "mani": [5, 27], "For": 5, "exampl": [5, 12, 24, 27], "fit": 5, "abl": [5, 18, 20, 21, 24, 25, 27, 28], "see": [5, 13, 14, 17], "specif": 5, "base": [5, 11], "model_factori": 5, "newmodel": 5, "const": 5, "new_model": 5, "y": [5, 18, 30], "arrai": 5, "_": 5, "rang": [5, 17], "save": [5, 16], "filepath": 5, "open": [5, 7], "w": 5, "f": [5, 18], "write": 5, "load": [5, 18, 20, 21, 24, 25, 27, 28], "r": [5, 8], "float": 5, "read": [5, 6, 9, 10], "doe": 5, "anyth": 5, "just": [5, 18], "intern": [5, 27, 28], "state": [5, 27, 28], "shap": 5, "until": [6, 9, 10], "refer": [6, 7, 9, 10], "an": [7, 17, 18, 27, 28], "compar": [7, 17, 27, 28], "broad": 7, "aim": 7, "aspect": 7, "interpret": 7, "instal": 7, "contribut": 7, "latest": 8, "version": 8, "packag": 8, "run": [8, 16, 17], "pip3": 8, "rememb": 8, "creat": [8, 16, 18, 20, 21, 24, 25, 27, 28], "virtual": 8, "environ": 8, "one": [8, 17, 27, 28], "after": 8, "verifi": 8, "print": [8, 17], "out": 8, "its": [8, 27, 28], "__version__": 8, "also": [8, 17, 18], "explainers_requir": 8, "txt": 8, "done": [8, 27, 28], "http": 8, "raw": 8, "githubusercont": 8, "com": 8, "oxid15": 8, "master": 8, "compactnesscas": [11, 19, 21], "continuitycas": [11, 19, 20, 21], "contrastivitycas": [11, 19, 21], "correctnesscas": [11, 19, 21], "covariatecomplexitycas": [11, 19, 21], "example_select": [11, 16, 19, 23, 26], "constantexplain": [11, 23, 24, 25], "limeexplain": [11, 17, 23, 25], "linearregressionexplain": [11, 23, 25], "randomexplain": [11, 23, 24, 25], "shapexplain": [11, 23, 25], "knnexplain": [11, 23, 24], "covariateregular": [11, 26, 27, 28], "batch_rms": [11, 30], "entropi": [11, 27, 28, 30], "batch_gini": [11, 30], "batch_count_eq": [11, 30], "minmax_norm": [11, 30], "avail": 12, "each": [12, 27, 28], "type": 12, "separ": 12, "featur": [12, 27, 28], "select": [12, 27], "scroll": [13, 14], "plot": [13, 14, 16], "try": 15, "knn": 16, "chosen": 16, "folder": 16, "have": [17, 18, 20, 21, 24, 25, 27, 28], "same": [17, 27, 28], "input": [17, 18, 27, 28], "output": 17, "allow": 17, "easili": 17, "lime_explain": [17, 25], "obtain": [17, 27, 28], "sampl": 17, "wrapper": 18, "around": 18, "collect": 18, "ml": 18, "infer": [18, 24, 25], "other": [18, 27, 28], "solut": 18, "given": [18, 27, 28], "certain": 18, "respons": 18, "call": [18, 20, 21, 24, 25, 27, 28], "successor": [18, 20, 21, 24, 25, 27, 28], "initi": [18, 20, 21, 24, 25, 27, 28], "meta": [18, 20, 21, 24, 25, 27, 28], "argument": [18, 20, 21, 24, 25, 27, 28], "relat": [18, 20, 21, 24, 25, 27, 28], "hyperparamet": [18, 20, 21, 24, 25, 27, 28], "architectur": [18, 20, 21, 24, 25, 27, 28], "addit": [18, 20, 21, 24, 25, 27, 28], "paramet": [18, 20, 21, 24, 25, 27, 28], "superclass": [18, 20, 21, 24, 25, 27, 28], "log": [18, 20, 21, 24, 25, 27, 28], "everyth": [18, 20, 21, 24, 25, 27, 28], "worth": [18, 20, 21, 24, 25, 27, 28], "about": [18, 20, 21, 24, 25, 27, 28], "wa": [18, 20, 21, 24, 25, 27, 28], "either": [18, 20, 21, 24, 25, 27, 28], "param": [18, 20, 21, 24, 25, 27, 28], "meta_prefix": [18, 20, 21, 24, 25, 27, 28], "encapsul": [18, 24, 25], "includ": [18, 24, 25], "preprocess": [18, 24, 25], "step": [18, 24, 25], "suffici": [18, 24, 25], "kind": [18, 27, 28], "e": 18, "g": 18, "anoth": [18, 27], "repres": 18, "high": 18, "properti": 18, "against": 18, "popul": 18, "noisy_d": [20, 21, 27, 28], "noisy_model": [21, 27, 28], "constant_explain": [24, 25], "knn_explain": 24, "random_explain": [24, 25], "constant": [25, 27, 28], "linear_regression_explain": 25, "get_coef": 25, "shift": 25, "magnitud": [25, 28], "shap_explain": 25, "covariate_regular": [27, 28], "covari": 28, "regular": 28, "over": [27, 28], "comprehens": 28, "averag": [27, 28], "simpl": [27, 28], "shannon": [27, 28], "normal": [27, 28], "The": [27, 28], "worst": [27, 28], "give": [27, 28], "equal": [27, 28], "n": [27, 28], "where": [27, 28], "number": [27, 28], "best": [27, 28], "maximum": [27, 28], "zero": [27, 28], "int": [27, 28, 30], "bx": 30, "bool": 30, "labeldiffer": [11, 26, 28], "otherdisagr": [11, 26, 28], "parameterrandomizationcheck": [11, 26, 27, 28], "smallnoisecheck": [11, 26, 27, 28], "sparsiti": [11, 26, 28], "targetdiscrimin": [11, 26, 27], "noisi": 27, "parameter_randomization_check": [27, 28], "check": [27, 28], "saniti": [27, 28], "ensur": [27, 28], "influenc": [27, 28], "follow": [27, 28], "chang": [27, 28], "expect": [27, 28], "stai": [27, 28], "baselin": [27, 28], "instead": [27, 28], "Then": [27, 28], "origin": [27, 28], "count": 27, "were": 27, "point": [27, 28], "so": [27, 28], "reach": [27, 28], "opposit": [27, 28], "distanc": [27, 28], "between": [27, 28], "maxim": [27, 28], "problem": [27, 28], "seem": [27, 28], "redund": [27, 28], "becaus": [27, 28], "mean": [27, 28], "correct": [27, 28], "difficult": [27, 28], "expl_noisy_kwarg": [27, 28], "small_noise_check": [27, 28], "natur": 27, "discret": 27, "choos": 27, "finit": 27, "rmse": [27, 28], "appropri": 27, "similar": 27, "suitabl": 27, "could": 27, "take": 27, "test": 27, "small": [27, 28], "amount": 27, "nois": [27, 28], "pair": 27, "didn": 27, "t": 27, "divid": 27, "total": 27, "ratio": 27, "continu": 27, "gener": 27, "slightli": 27, "target_discrimin": 27, "true": 27, "discrimin": 27, "describ": 27, "perform": 27, "adopt": 27, "imbalanc": 27, "f1": 27, "descript": 27, "insuffici": 27, "grasp": 27, "label_differ": 28, "target": 28, "In": 28, "binari": 28, "posit": 28, "neg": 28, "multiclass": 28, "classif": 28, "distant": 28, "expls_kwarg": 28, "whole": 28, "appli": 28, "perturb": 28, "huge": 28, "variat": 28, "achiev": 28, "gini": 28, "henc": 28, "most": 28, "unequ": 28, "distribut": 28, "sparsest": 28, "syntheticdataset": [11, 22], "synthetic_dataset": 22, "frac": 22}, "objects": {"xaib.base": [[18, 0, 1, "", "Case"], [18, 0, 1, "", "Dataset"], [18, 0, 1, "", "Explainer"], [18, 0, 1, "", "Model"]], "xaib.base.Case": [[18, 1, 1, "", "__init__"], [18, 1, 1, "", "add_metric"], [18, 1, 1, "", "evaluate"]], "xaib.base.Dataset": [[18, 1, 1, "", "__init__"]], "xaib.base.Explainer": [[18, 1, 1, "", "__init__"], [18, 1, 1, "", "predict"]], "xaib.base.Model": [[18, 1, 1, "", "__init__"], [18, 1, 1, "", "predict"]], "xaib.cases.example_selection": [[20, 0, 1, "", "ContinuityCase"]], "xaib.cases.example_selection.ContinuityCase": [[20, 1, 1, "", "__init__"]], "xaib.cases.feature_importance": [[21, 0, 1, "", "CoherenceCase"], [21, 0, 1, "", "CompactnessCase"], [21, 0, 1, "", "ContinuityCase"], [21, 0, 1, "", "ContrastivityCase"], [21, 0, 1, "", "CorrectnessCase"], [21, 0, 1, "", "CovariateComplexityCase"]], "xaib.cases.feature_importance.CoherenceCase": [[21, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.CompactnessCase": [[21, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.ContinuityCase": [[21, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.ContrastivityCase": [[21, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.CorrectnessCase": [[21, 1, 1, "", "__init__"]], "xaib.cases.feature_importance.CovariateComplexityCase": [[21, 1, 1, "", "__init__"]], "xaib.datasets.synthetic_dataset": [[22, 0, 1, "", "SyntheticDataset"]], "xaib.datasets.synthetic_dataset.SyntheticDataset": [[22, 1, 1, "", "__init__"]], "xaib.explainers.example_selection.constant_explainer": [[24, 0, 1, "", "ConstantExplainer"]], "xaib.explainers.example_selection.constant_explainer.ConstantExplainer": [[24, 1, 1, "", "__init__"], [24, 1, 1, "", "predict"]], "xaib.explainers.example_selection.knn_explainer": [[24, 0, 1, "", "KNNExplainer"]], "xaib.explainers.example_selection.knn_explainer.KNNExplainer": [[24, 1, 1, "", "__init__"], [24, 1, 1, "", "predict"]], "xaib.explainers.example_selection.random_explainer": [[24, 0, 1, "", "RandomExplainer"]], "xaib.explainers.example_selection.random_explainer.RandomExplainer": [[24, 1, 1, "", "__init__"], [24, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.constant_explainer": [[25, 0, 1, "", "ConstantExplainer"]], "xaib.explainers.feature_importance.constant_explainer.ConstantExplainer": [[25, 1, 1, "", "__init__"], [25, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.lime_explainer": [[25, 0, 1, "", "LimeExplainer"]], "xaib.explainers.feature_importance.lime_explainer.LimeExplainer": [[25, 1, 1, "", "__init__"], [25, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.linear_regression_explainer": [[25, 0, 1, "", "LinearRegressionExplainer"]], "xaib.explainers.feature_importance.linear_regression_explainer.LinearRegressionExplainer": [[25, 1, 1, "", "__init__"], [25, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.random_explainer": [[25, 0, 1, "", "RandomExplainer"]], "xaib.explainers.feature_importance.random_explainer.RandomExplainer": [[25, 1, 1, "", "__init__"], [25, 1, 1, "", "predict"]], "xaib.explainers.feature_importance.shap_explainer": [[25, 0, 1, "", "ShapExplainer"]], "xaib.explainers.feature_importance.shap_explainer.ShapExplainer": [[25, 1, 1, "", "__init__"], [25, 1, 1, "", "predict"]], "xaib.metrics.example_selection.covariate_regularity": [[27, 0, 1, "", "CovariateRegularity"]], "xaib.metrics.example_selection.covariate_regularity.CovariateRegularity": [[27, 1, 1, "", "__init__"], [27, 1, 1, "", "compute"]], "xaib.metrics.example_selection.parameter_randomization_check": [[27, 0, 1, "", "ParameterRandomizationCheck"]], "xaib.metrics.example_selection.parameter_randomization_check.ParameterRandomizationCheck": [[27, 1, 1, "", "__init__"], [27, 1, 1, "", "compute"]], "xaib.metrics.example_selection.small_noise_check": [[27, 0, 1, "", "SmallNoiseCheck"]], "xaib.metrics.example_selection.small_noise_check.SmallNoiseCheck": [[27, 1, 1, "", "__init__"], [27, 1, 1, "", "compute"]], "xaib.metrics.example_selection.target_discriminativeness": [[27, 0, 1, "", "TargetDiscriminativeness"]], "xaib.metrics.example_selection.target_discriminativeness.TargetDiscriminativeness": [[27, 1, 1, "", "__init__"], [27, 1, 1, "", "compute"]], "xaib.metrics.feature_importance.covariate_regularity": [[28, 0, 1, "", "CovariateRegularity"]], "xaib.metrics.feature_importance.covariate_regularity.CovariateRegularity": [[28, 1, 1, "", "__init__"], [28, 1, 1, "", "compute"]], "xaib.metrics.feature_importance.label_difference": [[28, 0, 1, "", "LabelDifference"]], "xaib.metrics.feature_importance.label_difference.LabelDifference": [[28, 1, 1, "", "__init__"], [28, 1, 1, "", "compute"]], "xaib.metrics.feature_importance.other_disagreement": [[28, 0, 1, "", "OtherDisagreement"]], "xaib.metrics.feature_importance.other_disagreement.OtherDisagreement": [[28, 1, 1, "", "__init__"], [28, 1, 1, "", "compute"]], "xaib.metrics.feature_importance.parameter_randomization_check": [[28, 0, 1, "", "ParameterRandomizationCheck"]], "xaib.metrics.feature_importance.parameter_randomization_check.ParameterRandomizationCheck": [[28, 1, 1, "", "__init__"], [28, 1, 1, "", "compute"]], "xaib.metrics.feature_importance.small_noise_check": [[28, 0, 1, "", "SmallNoiseCheck"]], "xaib.metrics.feature_importance.small_noise_check.SmallNoiseCheck": [[28, 1, 1, "", "__init__"], [28, 1, 1, "", "compute"]], "xaib.metrics.feature_importance.sparsity": [[28, 0, 1, "", "Sparsity"]], "xaib.metrics.feature_importance.sparsity.Sparsity": [[28, 1, 1, "", "__init__"], [28, 1, 1, "", "compute"]], "xaib.utils": [[30, 0, 1, "", "SimpleDataloader"], [30, 2, 1, "", "batch_count_eq"], [30, 2, 1, "", "batch_gini"], [30, 2, 1, "", "batch_rmse"], [30, 2, 1, "", "entropy"], [30, 2, 1, "", "minmax_normalize"]], "xaib.utils.SimpleDataloader": [[30, 1, 1, "", "__init__"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "function", "Python function"]}, "titleterms": {"case": [0, 15, 19, 20, 21], "how": 1, "contribut": 1, "add": [2, 3, 4, 5], "dataset": [2, 6, 22], "creat": [2, 3, 4, 5], "data": 2, "wrapper": [2, 3, 5], "test": [2, 3, 4, 5], "new": [2, 3, 4, 5], "integr": [2, 3, 4, 5], "explain": [3, 23, 24, 25], "metric": [4, 9, 26, 27, 28], "model": [5, 10, 29], "welcom": 7, "xaib": [7, 11, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "instal": 8, "modul": 11, "refer": 11, "evalu": [12, 16], "result": [12, 13, 14], "exampl": 13, "select": 13, "featur": 14, "import": 14, "us": 15, "method": [16, 17], "try": 17, "base": 18, "example_select": [20, 24, 27], "feature_import": [21, 25, 28], "util": 30}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.todo": 2, "sphinx.ext.viewcode": 1, "sphinx": 57}, "alltitles": {"Cases": [[0, "cases"]], "How to contribute": [[1, "how-to-contribute"]], "Add dataset": [[2, "add-dataset"]], "Create data wrapper": [[2, "create-data-wrapper"]], "Test new dataset": [[2, "test-new-dataset"]], "Integrate new dataset": [[2, "integrate-new-dataset"]], "Add explainer": [[3, "add-explainer"]], "Create explainer wrapper": [[3, "create-explainer-wrapper"]], "Test new explainer": [[3, "test-new-explainer"]], "Integrate new explainer": [[3, "integrate-new-explainer"]], "Add metric": [[4, "add-metric"]], "Create metric": [[4, "create-metric"]], "Test new metric": [[4, "test-new-metric"]], "Integrate new metric": [[4, "integrate-new-metric"]], "Add model": [[5, "add-model"]], "Create model wrapper": [[5, "create-model-wrapper"]], "Test new model": [[5, "test-new-model"]], "Integrate new model": [[5, "integrate-new-model"]], "Datasets": [[6, "datasets"]], "Welcome to XAIB!": [[7, "welcome-to-xaib"]], "Installation": [[8, "installation"]], "Metrics": [[9, "metrics"]], "Models": [[10, "models"]], "Evaluation results": [[12, "evaluation-results"]], "Example Selection Results": [[13, "example-selection-results"]], "Feature Importance Results": [[14, "feature-importance-results"]], "Use cases": [[15, "use-cases"]], "Evaluate a method": [[16, "evaluate-a-method"]], "Try a method": [[17, "try-a-method"]], "xaib.base": [[18, "xaib-base"]], "xaib.cases": [[19, "xaib-cases"]], "xaib.cases.example_selection": [[20, "xaib-cases-example-selection"]], "xaib.cases.feature_importance": [[21, "xaib-cases-feature-importance"]], "xaib.explainers": [[23, "xaib-explainers"]], "xaib.explainers.example_selection": [[24, "xaib-explainers-example-selection"]], "xaib.explainers.feature_importance": [[25, "xaib-explainers-feature-importance"]], "xaib.metrics": [[26, "xaib-metrics"]], "xaib.utils": [[30, "xaib-utils"]], "xaib.metrics.example_selection": [[27, "xaib-metrics-example-selection"]], "xaib.datasets": [[22, "xaib-datasets"]], "XAIB modules reference": [[11, "xaib-modules-reference"]], "xaib.models": [[29, "xaib-models"]], "xaib.metrics.feature_importance": [[28, "xaib-metrics-feature-importance"]]}, "indexentries": {"covariateregularity (class in xaib.metrics.feature_importance.covariate_regularity)": [[28, "xaib.metrics.feature_importance.covariate_regularity.CovariateRegularity"]], "labeldifference (class in xaib.metrics.feature_importance.label_difference)": [[28, "xaib.metrics.feature_importance.label_difference.LabelDifference"]], "otherdisagreement (class in xaib.metrics.feature_importance.other_disagreement)": [[28, "xaib.metrics.feature_importance.other_disagreement.OtherDisagreement"]], "parameterrandomizationcheck (class in xaib.metrics.feature_importance.parameter_randomization_check)": [[28, "xaib.metrics.feature_importance.parameter_randomization_check.ParameterRandomizationCheck"]], "smallnoisecheck (class in xaib.metrics.feature_importance.small_noise_check)": [[28, "xaib.metrics.feature_importance.small_noise_check.SmallNoiseCheck"]], "sparsity (class in xaib.metrics.feature_importance.sparsity)": [[28, "xaib.metrics.feature_importance.sparsity.Sparsity"]], "__init__() (xaib.metrics.feature_importance.covariate_regularity.covariateregularity method)": [[28, "xaib.metrics.feature_importance.covariate_regularity.CovariateRegularity.__init__"]], "__init__() (xaib.metrics.feature_importance.label_difference.labeldifference method)": [[28, "xaib.metrics.feature_importance.label_difference.LabelDifference.__init__"]], "__init__() (xaib.metrics.feature_importance.other_disagreement.otherdisagreement method)": [[28, "xaib.metrics.feature_importance.other_disagreement.OtherDisagreement.__init__"]], "__init__() (xaib.metrics.feature_importance.parameter_randomization_check.parameterrandomizationcheck method)": [[28, "xaib.metrics.feature_importance.parameter_randomization_check.ParameterRandomizationCheck.__init__"]], "__init__() (xaib.metrics.feature_importance.small_noise_check.smallnoisecheck method)": [[28, "xaib.metrics.feature_importance.small_noise_check.SmallNoiseCheck.__init__"]], "__init__() (xaib.metrics.feature_importance.sparsity.sparsity method)": [[28, "xaib.metrics.feature_importance.sparsity.Sparsity.__init__"]], "compute() (xaib.metrics.feature_importance.covariate_regularity.covariateregularity method)": [[28, "xaib.metrics.feature_importance.covariate_regularity.CovariateRegularity.compute"]], "compute() (xaib.metrics.feature_importance.label_difference.labeldifference method)": [[28, "xaib.metrics.feature_importance.label_difference.LabelDifference.compute"]], "compute() (xaib.metrics.feature_importance.other_disagreement.otherdisagreement method)": [[28, "xaib.metrics.feature_importance.other_disagreement.OtherDisagreement.compute"]], "compute() (xaib.metrics.feature_importance.parameter_randomization_check.parameterrandomizationcheck method)": [[28, "xaib.metrics.feature_importance.parameter_randomization_check.ParameterRandomizationCheck.compute"]], "compute() (xaib.metrics.feature_importance.small_noise_check.smallnoisecheck method)": [[28, "xaib.metrics.feature_importance.small_noise_check.SmallNoiseCheck.compute"]], "compute() (xaib.metrics.feature_importance.sparsity.sparsity method)": [[28, "xaib.metrics.feature_importance.sparsity.Sparsity.compute"]]}})
\ No newline at end of file
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index 75f4de5..d2ac151 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -1 +1 @@
-http://oxid15.github.io/xai-benchmark/cases.htmlhttp://oxid15.github.io/xai-benchmark/contribute.htmlhttp://oxid15.github.io/xai-benchmark/contribute/add_dataset.htmlhttp://oxid15.github.io/xai-benchmark/contribute/add_explainer.htmlhttp://oxid15.github.io/xai-benchmark/contribute/add_metric.htmlhttp://oxid15.github.io/xai-benchmark/contribute/add_model.htmlhttp://oxid15.github.io/xai-benchmark/datasets.htmlhttp://oxid15.github.io/xai-benchmark/index.htmlhttp://oxid15.github.io/xai-benchmark/installation.htmlhttp://oxid15.github.io/xai-benchmark/metrics.htmlhttp://oxid15.github.io/xai-benchmark/models.htmlhttp://oxid15.github.io/xai-benchmark/modules.htmlhttp://oxid15.github.io/xai-benchmark/results.htmlhttp://oxid15.github.io/xai-benchmark/use_cases.htmlhttp://oxid15.github.io/xai-benchmark/use_cases/evaluate_method.htmlhttp://oxid15.github.io/xai-benchmark/use_cases/try_method.htmlhttp://oxid15.github.io/xai-benchmark/xaib/base.htmlhttp://oxid15.github.io/xai-benchmark/xaib/cases.htmlhttp://oxid15.github.io/xai-benchmark/xaib/cases.example_selection.htmlhttp://oxid15.github.io/xai-benchmark/xaib/cases.feature_importance.htmlhttp://oxid15.github.io/xai-benchmark/xaib/explainers.htmlhttp://oxid15.github.io/xai-benchmark/xaib/explainers.example_selection.htmlhttp://oxid15.github.io/xai-benchmark/xaib/explainers.feature_importance.htmlhttp://oxid15.github.io/xai-benchmark/xaib/utils.htmlhttp://oxid15.github.io/xai-benchmark/genindex.htmlhttp://oxid15.github.io/xai-benchmark/_modules/index.htmlhttp://oxid15.github.io/xai-benchmark/search.html
\ No newline at end of file
+http://oxid15.github.io/xai-benchmark/index.htmlhttp://oxid15.github.io/xai-benchmark/xaib/metrics.htmlhttp://oxid15.github.io/xai-benchmark/xaib/metrics.feature_importance.htmlhttp://oxid15.github.io/xai-benchmark/genindex.htmlhttp://oxid15.github.io/xai-benchmark/_modules/xaib/metrics/feature_importance/covariate_regularity.htmlhttp://oxid15.github.io/xai-benchmark/_modules/index.htmlhttp://oxid15.github.io/xai-benchmark/search.html
\ No newline at end of file
diff --git a/docs/xaib/base.html b/docs/xaib/base.html
index 29f7693..b4403bb 100644
--- a/docs/xaib/base.html
+++ b/docs/xaib/base.html
@@ -1 +1 @@
-                        xaib.base — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.base
 
 
 class xaib.base.Dataset(*args: Any, **kwargs: Any)[source] 
Dataset is a wrapper around any collection of items to put in the ML model for inference
 
 
 __init__(*args: Any, **kwargs: Any) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.base.Model(*args: Any, **kwargs: Any)[source] 
Model is a wrapper around any inference of ML or other solution in the form y = f(x) it implements method predict that given certain data x returns the response y
 
 
 __init__(*args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x: Any) → Any[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.base.Explainer(*args: Any, **kwargs: Any)[source] 
Explainer is a special kind of Model e = g(f, x) that accepts another Model and data as input and also returns a response e - an explanation
 
 
 __init__(*args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x: Any, model: Model) → Any[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.base.Case(*args: Any, **kwargs: Any)[source] 
Case is a collection of Metrics which represent some high-level property of an Explainer
 
 
 __init__(*args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 add_metric(name: str, metric: Metric) → None[source] 
 
 
 evaluate(name: str, expl: Explainer, metrics_kwargs: Dict[str, Dict[Any, Any]] | None = None, **kwargs: Any) → None[source] 
Evaluates model against any metrics. Should not return any value, just populate self.metrics dict.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
+                        xaib.base — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.base
 
 
 class xaib.base.Dataset(*args: Any, **kwargs: Any)[source] 
Dataset is a wrapper around any collection of items to put in the ML model for inference
 
 
 __init__(*args: Any, **kwargs: Any) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.base.Model(*args: Any, **kwargs: Any)[source] 
Model is a wrapper around any inference of ML or other solution in the form y = f(x) it implements method predict that given certain data x returns the response y
 
 
 __init__(*args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x: Any) → Any[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.base.Explainer(*args: Any, **kwargs: Any)[source] 
Explainer is a special kind of Model e = g(f, x) that accepts another Model and data as input and also returns a response e - an explanation
 
 
 __init__(*args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x: Any, model: Model) → Any[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.base.Case(*args: Any, **kwargs: Any)[source] 
Case is a collection of Metrics which represent some high-level property of an Explainer
 
 
 __init__(*args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 add_metric(name: str, metric: Metric) → None[source] 
 
 
 evaluate(name: str, expl: Explainer, metrics_kwargs: Dict[str, Dict[Any, Any]] | None = None, **kwargs: Any) → None[source] 
Evaluates model against any metrics. Should not return any value, just populate self.metrics dict.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/cases.example_selection.html b/docs/xaib/cases.example_selection.html
index 8c0c6bc..092b256 100644
--- a/docs/xaib/cases.example_selection.html
+++ b/docs/xaib/cases.example_selection.html
@@ -1 +1 @@
-                        xaib.cases.example_selection — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.cases.example_selection
 
 
 class xaib.cases.example_selection.ContinuityCase(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
+                        xaib.cases.example_selection — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.cases.example_selection
 
 
 class xaib.cases.example_selection.ContinuityCase(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/cases.feature_importance.html b/docs/xaib/cases.feature_importance.html
index 699f550..cfa69d8 100644
--- a/docs/xaib/cases.feature_importance.html
+++ b/docs/xaib/cases.feature_importance.html
@@ -1 +1 @@
-                        xaib.cases.feature_importance — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.cases.feature_importance
 
 
 class xaib.cases.feature_importance.CoherenceCase(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.CompactnessCase(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.ContinuityCase(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.ContrastivityCase(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.CorrectnessCase(ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.CovariateComplexityCase(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
+                        xaib.cases.feature_importance — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.cases.feature_importance
 
 
 class xaib.cases.feature_importance.CoherenceCase(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.CompactnessCase(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.ContinuityCase(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.ContrastivityCase(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.CorrectnessCase(ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.cases.feature_importance.CovariateComplexityCase(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/cases.html b/docs/xaib/cases.html
index 20202d6..0aff67e 100644
--- a/docs/xaib/cases.html
+++ b/docs/xaib/cases.html
@@ -1 +1 @@
-                        xaib.cases — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
+                        xaib.cases — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/datasets.html b/docs/xaib/datasets.html
new file mode 100644
index 0000000..e92936f
--- /dev/null
+++ b/docs/xaib/datasets.html
@@ -0,0 +1 @@
+                        xaib.datasets — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.datasets
 
 
 class xaib.datasets.synthetic_dataset.SyntheticDataset(split, name=None, frac=0.8, *args, **kwargs)[source] 
 
 __init__(split, name=None, frac=0.8, *args, **kwargs) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/explainers.example_selection.html b/docs/xaib/explainers.example_selection.html
index 144ce53..6f8fc58 100644
--- a/docs/xaib/explainers.example_selection.html
+++ b/docs/xaib/explainers.example_selection.html
@@ -1 +1 @@
-                        xaib.explainers.example_selection — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.explainers.example_selection
 
 
 class xaib.explainers.example_selection.constant_explainer.ConstantExplainer(train_ds, example, **kwargs)[source] 
 
 __init__(train_ds, example, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.example_selection.knn_explainer.KNNExplainer(train_ds, **kwargs)[source] 
 
 __init__(train_ds, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.example_selection.random_explainer.RandomExplainer(train_ds, **kwargs)[source] 
 
 __init__(train_ds, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
+                        xaib.explainers.example_selection — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.explainers.example_selection
 
 
 class xaib.explainers.example_selection.constant_explainer.ConstantExplainer(train_ds, example, **kwargs)[source] 
 
 __init__(train_ds, example, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.example_selection.knn_explainer.KNNExplainer(train_ds, **kwargs)[source] 
 
 __init__(train_ds, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.example_selection.random_explainer.RandomExplainer(train_ds, **kwargs)[source] 
 
 __init__(train_ds, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/explainers.feature_importance.html b/docs/xaib/explainers.feature_importance.html
index df2e616..7b76f80 100644
--- a/docs/xaib/explainers.feature_importance.html
+++ b/docs/xaib/explainers.feature_importance.html
@@ -1 +1 @@
-                        xaib.explainers.feature_importance — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.explainers.feature_importance
 
 
 class xaib.explainers.feature_importance.constant_explainer.ConstantExplainer(constant: Any, *args: Any, **kwargs: Any)[source] 
 
 __init__(constant: Any, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x: Any, model: Model) → Any[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.feature_importance.lime_explainer.LimeExplainer(train_ds, labels, *args, meta_prefix=None, **kwargs)[source] 
 
 __init__(train_ds, labels, *args, meta_prefix=None, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.feature_importance.linear_regression_explainer.LinearRegressionExplainer(get_coef)[source] 
 
 __init__(get_coef) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.feature_importance.random_explainer.RandomExplainer(shift=0, magnitude=1, *args, **kwargs)[source] 
 
 __init__(shift=0, magnitude=1, *args, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x: Any, model: Model) → Any[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.feature_importance.shap_explainer.ShapExplainer(train_ds, *args, meta_prefix=None, **kwargs)[source] 
 
 __init__(train_ds, *args, meta_prefix=None, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
+                        xaib.explainers.feature_importance — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.explainers.feature_importance
 
 
 class xaib.explainers.feature_importance.constant_explainer.ConstantExplainer(constant: Any, *args: Any, **kwargs: Any)[source] 
 
 __init__(constant: Any, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x: Any, model: Model) → Any[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.feature_importance.lime_explainer.LimeExplainer(train_ds, labels, *args, meta_prefix=None, **kwargs)[source] 
 
 __init__(train_ds, labels, *args, meta_prefix=None, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.feature_importance.linear_regression_explainer.LinearRegressionExplainer(get_coef)[source] 
 
 __init__(get_coef) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.feature_importance.random_explainer.RandomExplainer(shift=0, magnitude=1, *args, **kwargs)[source] 
 
 __init__(shift=0, magnitude=1, *args, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x: Any, model: Model) → Any[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.explainers.feature_importance.shap_explainer.ShapExplainer(train_ds, *args, meta_prefix=None, **kwargs)[source] 
 
 __init__(train_ds, *args, meta_prefix=None, **kwargs) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 predict(x, model)[source] 
Method to encapsulate inference. May include preprocessing steps to make model self-sufficient.
 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/explainers.html b/docs/xaib/explainers.html
index bd308de..07da8b3 100644
--- a/docs/xaib/explainers.html
+++ b/docs/xaib/explainers.html
@@ -1 +1 @@
-                        xaib.explainers — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.explainers
 
  
 
 
 
 
 
 
   
\ No newline at end of file
+                        xaib.explainers — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.explainers
 
  
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/metrics.example_selection.html b/docs/xaib/metrics.example_selection.html
new file mode 100644
index 0000000..2039c0a
--- /dev/null
+++ b/docs/xaib/metrics.example_selection.html
@@ -0,0 +1 @@
+                        xaib.metrics.example_selection — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.metrics.example_selection
 
 
 class xaib.metrics.example_selection.covariate_regularity.CovariateRegularity(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
Measures how noisy the features in the examples provided by the method.
 
More simple explanations are considered better. This is measured by average Shannon entropy over batch-normalized explanations.
 
The less the better
 
 
     
  • Worst case: constant explainer that gives examples with same importance to each feature, that is equal to 1/N where N is the number of features
     
  • Best case: constant explainer that gives examples with one feature with maximum value and others zero
     
 
 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(expl: Explainer, batch_size: int = 1, expl_kwargs: Dict[Any, Any] | None = None) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.metrics.example_selection.parameter_randomization_check.ParameterRandomizationCheck(ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any)[source] 
Parameter randomization check is a sanity-check. To ensure that the model influence explanations the following is done. The model is changed and it is expected that explanations should not stay the same is model changed. This check uses random model baselines instead of same models with randomized internal states. Then the explanations on the original data are obtained. They are compared with explanations done with the original model by counting how many examples were the same for same data points. The less the better.
 
 
The less the better
     
  • Worst case: explanations are the same, so it is Constant explainer
     
  • Best case: is reached when explanations are the opposite,
     
 
distance between them maximized.
 
 
The problem with this kind of metric is with its maximization. It seems redundant to maximize it because more different explanations on random states do not mean that the model is more correct. It is difficult to define best case explainer in this case - the metric has no maximum value.
 
 
 __init__(ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(expl: Explainer, batch_size: int = 1, expl_kwargs: Dict[Any, Any] | None = None, expl_noisy_kwargs: Dict[Any, Any] | None = None) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.metrics.example_selection.small_noise_check.SmallNoiseCheck(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
Since the nature of example selection methods is discrete - we choose an example from finite set, the use of RMSE measure may be not appropriate. This means that the metric that is similar to the one that was used in feature importance case is not suitable. More appropriate metric could be the following: take the test dataset and add small amount of noise to the items as was done in the feature importance case. Then count the number of item pairs when the example provided didn’t change and divide by the total number of items. This ratio then is how continuous example generator is - if it provides the same examples for slightly changed inputs, then it is continuous.
 
 
The less the better
     
  • Worst case: Constant explainer
     
  • Best case: Random explainer
     
 
 
 
 __init__(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(expl: Explainer, batch_size: int = 1, expl_kwargs: Dict[Any, Any] | None = None) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.metrics.example_selection.target_discriminativeness.TargetDiscriminativeness(ds, model, *args, **kwargs)[source] 
Given true labels and explanations in form of the examples, train another model to discriminate between labels. The quality of the model on the examples given describes the quality of the explanations. The quality can be measured by any performance metric, but it is better to adopt to imbalanced data and use F1-measure for example.
 
The greater the better
 
Best case: examples are descriptive of labels so that the model reaches best performance Worst case: constant or random baseline - giving insufficient information to grasp labels
 
 
 __init__(ds, model, *args, **kwargs)[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(explainer, batch_size=1, expl_kwargs=None)[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/metrics.feature_importance.html b/docs/xaib/metrics.feature_importance.html
new file mode 100644
index 0000000..8572363
--- /dev/null
+++ b/docs/xaib/metrics.feature_importance.html
@@ -0,0 +1 @@
+                        xaib.metrics.feature_importance — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.metrics.feature_importance
 
 
 class xaib.metrics.feature_importance.covariate_regularity.CovariateRegularity(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
Covariate Regularity using entropy over explanations
 
This measures how comprehensible the explanations are in average. More simple explanations are considered better. This is measured by average Shannon entropy over batch-normalized explanations.
 
 
The less the better
     
  • Worst case: constant explainer that gives same importance to each feature, that is equal to 1/N where N is the number of features
     
  • Best case: constant explainer that gives one feature maximum value and others zero
     
 
 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(expl: Explainer, batch_size: int = 1, expl_kwargs: Dict[Any, Any] | None = None)[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.metrics.feature_importance.label_difference.LabelDifference(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
Obtain explanations for all targets and compare them. In binary case one metric can be computed - the difference between explanations of positive and negative targets. In case of multiclass classification we can compare one explanation to explanations of all other classes and obtain a number of metrics which can be averaged.
 
The more the better
 
     
  • Worst case: same explanations for different targets - constant explainer
     
  • Best case: different explanations for different targets
     
 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(expl: Explainer, batch_size: int = 1, expl_kwargs: Dict[Any, Any] | None = None) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.metrics.feature_importance.other_disagreement.OtherDisagreement(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
Measures how distant explanations on the same data points for this particular method from explanations of all others. Average RMSE is used as a metric. The less the better
 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(expl: Explainer, batch_size: int = 1, expls: List[Explainer] | None = None, expl_kwargs: Dict[Any, Any] | None = None, expls_kwargs: List[Dict[Any, Any]] | None = None) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.metrics.feature_importance.parameter_randomization_check.ParameterRandomizationCheck(ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any)[source] 
Parameter randomization check is a sanity-check. To ensure that the model influence explanations the following is done. The model is changed and it is expected that explanations should not stay the same is model changed. This check uses random model baselines instead of same models with randomized internal states. Then the explanations on the original data are obtained. They are compared with explanations done with the original model using average RMSE on the whole dataset. The further original explanations from the explanations on the randomized model the better.
 
 
The less the better
     
  • Worst case: explanations are the same, so it is Constant explainer.
     
  • Best case: is reached when explanations are the opposite, distance between them maximized. The problem with this kind of metric is with its maximization. It seems redundant to maximize it because more different explanations on random states do not mean that the model is more correct.
     
 
 
It is difficult to define best case explainer in this case - the metric has no maximum value.
 
 
 __init__(ds: Dataset, model: Model, noisy_model: Model, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(expl: Explainer, batch_size: int = 1, expl_kwargs: Dict[Any, Any] | None = None, expl_noisy_kwargs: Dict[Any, Any] | None = None) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.metrics.feature_importance.small_noise_check.SmallNoiseCheck(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
Apply noise of small magnitude to the input data. Obtain original and perturbed explanations. Compare them using RMSE and average. The less the better
 
 
     
  • Worst case: is when explanations are hugely changed by the small variations in input
     
  • Best case: is no variations, so constant explainer should achieve best results
     
 
 
 
 __init__(ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(expl: Explainer, batch_size: int = 1, expl_kwargs: Dict[Any, Any] | None = None) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 class xaib.metrics.feature_importance.sparsity.Sparsity(ds: Dataset, model: Model, *args: Any, **kwargs: Any)[source] 
Considering Gini-index as a measure of sparsity, one can give an average of it as a measure of sparsity for explanations. The greater the better
 
 
     
  • Worst case: is achieved by constant explainer that gives same
     
 
importance to each feature that is equal to 1/N where N is the number of features will obtain best gini index and hence worst sparsity - Best case: is when explainer is constant and gives one feature maximum value and others zero, which is the most unequal distribution and is the sparsest explanation that can be given
 
 
 
 __init__(ds: Dataset, model: Model, *args: Any, **kwargs: Any) → None[source] 
Should be called in any successor - initializes default meta needed. Arguments passed in it should be related to model’s hyperparameters, architecture. All additional arguments should have defaults - to be able to create model and then load. Successors may pass all of their parameters to superclass for it to be able to log them in meta. Everything that is worth to document about model and data it was trained on can be either in params or meta_prefix.
 
 
 
 compute(expl: Explainer, batch_size: int = 1, expl_kwargs: Dict[Any, Any] | None = None) → None[source] 
 
 
 

 
 
 

 
 
 

 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/metrics.html b/docs/xaib/metrics.html
new file mode 100644
index 0000000..194d839
--- /dev/null
+++ b/docs/xaib/metrics.html
@@ -0,0 +1 @@
+                        xaib.metrics — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.metrics
 
  
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/models.html b/docs/xaib/models.html
new file mode 100644
index 0000000..776fa2d
--- /dev/null
+++ b/docs/xaib/models.html
@@ -0,0 +1 @@
+                        xaib.models — xai-benchmark 0.2.0-alpha documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
xaib.models
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/xaib/utils.html b/docs/xaib/utils.html
index d742200..7b0205b 100644
--- a/docs/xaib/utils.html
+++ b/docs/xaib/utils.html
@@ -1 +1 @@
-                        xaib.utils — xai-benchmark 0.2.0-alpha documentation                      Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
+                        xaib.utils — xai-benchmark 0.2.0-alpha documentation                      Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file