Merge pull request #18 from Oxid15/develop

v0.2.0

setup.py

@@ -1,30 +1,24 @@
 import setuptools
 
-
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 setuptools.setup(
     name="xai-benchmark",
-    version='0.2.0-alpha',
-    author='Ilia Moiseev',
-    author_email='[email protected]',
-    license='MIT',
+    version="0.2.0",
+    author="Ilia Moiseev",
+    author_email="[email protected]",
+    license="MIT",
     description="Benchmark for Explainable AI methods",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/oxid15/xai-benchmark",
     classifiers=[
         "Programming Language :: Python :: 3",
-        "Operating System :: OS Independent"
+        "Operating System :: OS Independent",
     ],
     package_dir={"xaib": "./xaib"},
     packages=setuptools.find_packages(),
     python_requires=">=3.8",
-    install_requires=[
-        'cascade-ml>=0.7.2',
-        'scikit-learn',
-        'plotly',
-        'kaleido'
-    ]
-)
+    install_requires=["cascade-ml>=0.7.2", "scikit-learn", "plotly", "kaleido"],
+)

xaib/__init__.py

@@ -1,6 +1,6 @@
-__version__ = '0.2.0-alpha'
-__author__ = 'Ilia Moiseev'
-__author_email__ = '[email protected]'
+__version__ = "0.2.0"
+__author__ = "Ilia Moiseev"
+__author_email__ = "[email protected]"
 
 
-from .base import Dataset, Model, Explainer, Metric, Case, Factory
+from .base import Case, Dataset, Explainer, Factory, Metric, Model

xaib/base/__init__.py

@@ -1 +1 @@
-from .base import Dataset, Model, Explainer, Metric, Case, Factory
+from .base import Case, Dataset, Explainer, Factory, Metric, Model

xaib/base/base.py

@@ -8,6 +8,7 @@ class Dataset(cdd.Dataset):
     Dataset is a wrapper around any collection of items to put in the ML model
     for inference
     """
+
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -21,6 +22,7 @@ class Model(cdm.Model):
     Model is a wrapper around any inference of ML or other solution in the form y = f(x)
     it implements method `predict` that given certain data x returns the response y
     """
+
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -34,6 +36,7 @@ class Explainer(cdm.Model):
     Explainer is a special kind of Model e = g(f, x) that accepts another Model and data as input
     and also returns a response e - an explanation
     """
+
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -47,32 +50,38 @@ class Metric(cdm.Model):
     Metric is an entity which accepts Explainer, Model and Dataset and outputs a metric
     corresponding to the quality of Explainer v = m(g, f, x)
     """
+
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
         self.direction = None
         self._ds = ds
         self._model = model
 
-    def compute(self, expl: Explainer, batch_size: int = 1, expl_kwargs: Union[Dict[Any, Any], None] = None) -> Any:
+    def compute(
+        self,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+    ) -> Any:
         raise NotImplementedError()
 
     def evaluate(
-            self,
-            name: str,
-            expl: Explainer,
-            batch_size: int = 1,
-            expl_kwargs: Union[Dict[Any, Any], None] = None,
-            **kwargs: Any
+        self,
+        name: str,
+        expl: Explainer,
+        batch_size: int = 1,
+        expl_kwargs: Union[Dict[Any, Any], None] = None,
+        **kwargs: Any
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
         value = self.compute(expl, batch_size=batch_size, **expl_kwargs, **kwargs)
 
-        self.params['name'] = name
-        self.params['direction'] = self.direction
-        self.params['dataset'] = self._ds.name
-        self.params['model'] = self._model.name
+        self.params["name"] = name
+        self.params["direction"] = self.direction
+        self.params["dataset"] = self._ds.name
+        self.params["model"] = self._model.name
         self.metrics[self.name] = value
 
 
@@ -81,6 +90,7 @@ class Case(cdm.Model):
     Case is a collection of Metrics which represent some
     high-level property of an Explainer
     """
+
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
@@ -89,21 +99,27 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
     def add_metric(self, name: str, metric: Metric) -> None:
         self._metric_objs[name] = metric
 
-    def evaluate(self, name: str, expl: Explainer, metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None, **kwargs: Any) -> None:
+    def evaluate(
+        self,
+        name: str,
+        expl: Explainer,
+        metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None,
+        **kwargs: Any
+    ) -> None:
         if metrics_kwargs is None:
             metrics_kwargs = {name: {} for _ in self._metric_objs}
 
-        self.params['metric_params'] = dict()
+        self.params["metric_params"] = dict()
         for m_name in self._metric_objs:
             mkwargs = {}
             if m_name in metrics_kwargs:
                 mkwargs = metrics_kwargs[m_name]
 
             self._metric_objs[m_name].evaluate(name, expl, **mkwargs, **kwargs)
 
-            self.params['case'] = self.name
+            self.params["case"] = self.name
 
-            self.params['metric_params'][m_name] = (self._metric_objs[m_name].params)
+            self.params["metric_params"][m_name] = self._metric_objs[m_name].params
             self.metrics.update(self._metric_objs[m_name].metrics)
 
 
@@ -112,15 +128,13 @@ class Factory:
     Collection of constructors to build and return objects
     from predefined hardcoded or added dynamically constructors
     """
+
     def __init__(self) -> None:
         self._constructors = dict()
         self._constructors_kwargs = dict()
 
     def _get_all(self) -> Dict[str, Any]:
-        return {
-                name: self._get(name)
-                for name in self._constructors
-            }
+        return {name: self._get(name) for name in self._constructors}
 
     def _get(self, name: str) -> Any:
         if name not in self._constructors_kwargs:
@@ -132,10 +146,15 @@ def _get(self, name: str) -> Any:
         return constructor(**kwargs)
 
     def get(self, name: str) -> Union[Dict[str, Any], Any]:
-        if name == 'all':
+        if name == "all":
             return self._get_all()
         return self._get(name)
 
-    def add(self, name: str, constructor: Callable[[Any], Any], constr_kwargs: Union[Any, None] = None) -> None:
+    def add(
+        self,
+        name: str,
+        constructor: Callable[[Any], Any],
+        constr_kwargs: Union[Any, None] = None,
+    ) -> None:
         self._constructors[name] = constructor
         self._constructors_kwargs[name] = constr_kwargs

xaib/cases/example_selection/__init__.py

@@ -1,4 +1,4 @@
 from .continuity_case import ContinuityCase
 from .contrastivity_case import ContrastivityCase
-from .covariate_complexity_case import CovariateComplexityCase
 from .correctness_case import CorrectnessCase
+from .covariate_complexity_case import CovariateComplexityCase

xaib/cases/example_selection/continuity_case.py

@@ -1,10 +1,13 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.example_selection import SmallNoiseCheck
 
 
 class ContinuityCase(Case):
-    def __init__(self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+    def __init__(
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'continuity'
-        self._metric_objs['small_noise_check'] = SmallNoiseCheck(ds, noisy_ds, model)
+        self.name = "continuity"
+        self._metric_objs["small_noise_check"] = SmallNoiseCheck(ds, noisy_ds, model)

xaib/cases/example_selection/contrastivity_case.py

@@ -1,10 +1,13 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.example_selection import TargetDiscriminativeness
 
 
 class ContrastivityCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'contrastivity'
-        self._metric_objs['target_discriminativeness'] = TargetDiscriminativeness(ds, model)
+        self.name = "contrastivity"
+        self._metric_objs["target_discriminativeness"] = TargetDiscriminativeness(
+            ds, model
+        )

xaib/cases/example_selection/correctness_case.py

@@ -1,10 +1,15 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.example_selection import ParameterRandomizationCheck
 
 
 class CorrectnessCase(Case):
-    def __init__(self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any) -> None:
+    def __init__(
+        self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'correctness'
-        self._metric_objs['parameter_randomization_check'] = ParameterRandomizationCheck(ds, model, noisy_model)
+        self.name = "correctness"
+        self._metric_objs[
+            "parameter_randomization_check"
+        ] = ParameterRandomizationCheck(ds, model, noisy_model)

xaib/cases/example_selection/covariate_complexity_case.py

@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.example_selection import CovariateRegularity
 
 
 class CovariateComplexityCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'covariate_complexity'
-        self._metric_objs['covariate_regularity'] = CovariateRegularity(ds, model)
+        self.name = "covariate_complexity"
+        self._metric_objs["covariate_regularity"] = CovariateRegularity(ds, model)

xaib/cases/feature_importance/__init__.py

@@ -1,6 +1,6 @@
-from .correctness_case import CorrectnessCase
+from .coherence_case import CoherenceCase
+from .compactness_case import CompactnessCase
 from .continuity_case import ContinuityCase
 from .contrastivity_case import ContrastivityCase
-from .coherence_case import CoherenceCase
+from .correctness_case import CorrectnessCase
 from .covariate_complexity_case import CovariateComplexityCase
-from .compactness_case import CompactnessCase

xaib/cases/feature_importance/coherence_case.py

@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import OtherDisagreement
 
 
 class CoherenceCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'coherence'
-        self._metric_objs['other_disagreement'] = OtherDisagreement(ds, model)
+        self.name = "coherence"
+        self._metric_objs["other_disagreement"] = OtherDisagreement(ds, model)

xaib/cases/feature_importance/compactness_case.py

@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import Sparsity
 
 
 class CompactnessCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'compactness'
-        self._metric_objs['sparsity'] = Sparsity(ds, model)
+        self.name = "compactness"
+        self._metric_objs["sparsity"] = Sparsity(ds, model)

xaib/cases/feature_importance/continuity_case.py

@@ -1,10 +1,13 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import SmallNoiseCheck
 
 
 class ContinuityCase(Case):
-    def __init__(self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
+    def __init__(
+        self, ds: Dataset, noisy_ds: Dataset, model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'continuity'
-        self._metric_objs['small_noise_check'] = SmallNoiseCheck(ds, noisy_ds, model)
+        self.name = "continuity"
+        self._metric_objs["small_noise_check"] = SmallNoiseCheck(ds, noisy_ds, model)

xaib/cases/feature_importance/contrastivity_case.py

@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import LabelDifference
 
 
 class ContrastivityCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'contrastivity'
-        self._metric_objs['label_difference'] = LabelDifference(ds, model)
+        self.name = "contrastivity"
+        self._metric_objs["label_difference"] = LabelDifference(ds, model)

xaib/cases/feature_importance/correctness_case.py

@@ -1,10 +1,15 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import ParameterRandomizationCheck
 
 
 class CorrectnessCase(Case):
-    def __init__(self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any) -> None:
+    def __init__(
+        self, ds: Dataset, model: Model, noisy_model: Model, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'correctness'
-        self._metric_objs['parameter_randomization_check'] = ParameterRandomizationCheck(ds, model, noisy_model)
+        self.name = "correctness"
+        self._metric_objs[
+            "parameter_randomization_check"
+        ] = ParameterRandomizationCheck(ds, model, noisy_model)

xaib/cases/feature_importance/covariate_complexity_case.py

@@ -1,10 +1,11 @@
 from typing import Any
-from ...base import Dataset, Model, Case
+
+from ...base import Case, Dataset, Model
 from ...metrics.feature_importance import CovariateRegularity
 
 
 class CovariateComplexityCase(Case):
     def __init__(self, ds: Dataset, model: Model, *args: Any, **kwargs: Any) -> None:
         super().__init__(ds, model, *args, **kwargs)
-        self.name = 'covariate_complexity'
-        self._metric_objs['covariate_regularity'] = CovariateRegularity(ds, model)
+        self.name = "covariate_complexity"
+        self._metric_objs["covariate_regularity"] = CovariateRegularity(ds, model)