Fix: Add an auto_format parameter to the ValidationResultSet constructor (#62)

Author: adrien-berchet

data_validation_framework/result.py

@@ -61,14 +61,9 @@ class ValidationResultSet(pd.DataFrame):
         "exception": None,
     }
 
-    def __init__(self, *args, output_columns=None, **kwargs):
+    def __init__(self, *args, output_columns=None, auto_format=False, **kwargs):
         super().__init__(*args, **kwargs)
-
-        # Skip weird states due to Pandas and tqdm interactions
-        data = kwargs.get("data", args[0] if len(args) > 0 else None)
-        if not isinstance(
-            data, (ValidationResultSet, pd.core.internals.managers.BlockManager)
-        ) and not set(self.index).intersection(set(ValidationResultSet.out_cols.keys())):
+        if auto_format:
             self.format_data(output_columns)
 
     @property

data_validation_framework/task.py

@@ -573,7 +573,7 @@ def run(self):
             )
 
         # Format the DataFrame
-        df = ValidationResultSet(new_df, output_columns=self.output_columns)
+        df = ValidationResultSet(new_df, output_columns=self.output_columns, auto_format=True)
 
         # Copy current index
         index = df.index.copy()
@@ -703,7 +703,7 @@ def _process(self, df, *args, **kwargs):
         df.loc[df["comment"].isnull(), "comment"] = ""
         df.loc[df["exception"].isnull(), "exception"] = ""
 
-        return df
+        return ValidationResultSet(df, auto_format=True)
 
 
 class ValidationWorkflow(SetValidationTask, luigi.WrapperTask):

setup.py

@@ -14,6 +14,7 @@
     "tqdm>=4.40",
 ]
 doc_reqs = [
+    "docutils<0.21",  # Temporary fix for m2r2
     "m2r2",
     "sphinx",
     "sphinx-bluebrain-theme",

tests/test_result.py

@@ -34,7 +34,7 @@ class TestValidationResultSet:
 
     def test_defaults(self):
         """Check defaults."""
-        df = result.ValidationResultSet(index=["a", "b"])
+        df = result.ValidationResultSet(index=["a", "b"], auto_format=True)
         assert df.to_dict() == {
             "is_valid": {"a": True, "b": True},
             "ret_code": {"a": 0, "b": 0},
@@ -44,7 +44,7 @@ def test_defaults(self):
 
     def test_defaults_with_extra_data(self):
         """Check defaults with extra data."""
-        df = result.ValidationResultSet({"a": [1, 2], "b": [3, 4]})
+        df = result.ValidationResultSet({"a": [1, 2], "b": [3, 4]}, auto_format=True)
         assert df.to_dict() == {
             "is_valid": {0: True, 1: True},
             "ret_code": {0: 0, 1: 0},
@@ -56,7 +56,7 @@ def test_defaults_with_extra_data(self):
 
     def test_default_ret_code(self):
         """Check default return code."""
-        df = result.ValidationResultSet({"is_valid": [True, False, True, False]})
+        df = result.ValidationResultSet({"is_valid": [True, False, True, False]}, auto_format=True)
         assert df.to_dict() == {
             "is_valid": {0: True, 1: False, 2: True, 3: False},
             "ret_code": {0: 0, 1: 1, 2: 0, 3: 1},
@@ -67,7 +67,9 @@ def test_default_ret_code(self):
     def test_output_columns(self):
         """Check output_columns argument."""
         df = result.ValidationResultSet(
-            index=["a", "b"], output_columns={"col1": "val1", "col2": "val2"}
+            index=["a", "b"],
+            output_columns={"col1": "val1", "col2": "val2"},
+            auto_format=True,
         )
         assert df.to_dict() == {
             "is_valid": {"a": True, "b": True},
@@ -84,6 +86,7 @@ def test_output_columns_with_existing_columns(self):
             {"is_valid": [True, False], "col1": ["test1", "test2"]},
             index=["a", "b"],
             output_columns={"col1": "val1", "col2": "val2"},
+            auto_format=True,
         )
         assert df.to_dict() == {
             "is_valid": {"a": True, "b": False},
@@ -126,9 +129,11 @@ def test_column_order(self):
             {"is_valid": [True, False], "col1": ["test1", "test2"]},
             index=["a", "b"],
             output_columns={"col1": "val1", "col2": "val2"},
+            auto_format=True,
         )
         df_order = result.ValidationResultSet(
-            pd.DataFrame(df[["col2", "comment", "is_valid", "col1", "exception", "ret_code"]])
+            pd.DataFrame(df[["col2", "comment", "is_valid", "col1", "exception", "ret_code"]]),
+            auto_format=True,
         )
         assert df_order.columns.tolist() == [
             "is_valid",
@@ -153,7 +158,8 @@ def test_column_order(self):
                         "ret_code",
                     ]
                 ]
-            )
+            ),
+            auto_format=True,
         )
         assert df_order.columns.tolist() == [
             "is_valid",
@@ -171,6 +177,7 @@ def test_pandas_method(self):
             {"is_valid": [True, False], "col1": ["test1", "test2"]},
             index=["a", "b"],
             output_columns={"col1": "val1", "col2": "val2"},
+            auto_format=True,
         )
         new_df = result.ValidationResultSet(df)
         assert new_df.apply(lambda x: x.ret_code * 5, axis=1).to_dict() == {"a": 0, "b": 5}

tests/test_task.py

@@ -1975,6 +1975,33 @@ def inputs(self):
         assert (tmpdir / "TestWorkflow" / "report.csv").exists()
         assert not (tmpdir / "report.pdf").exists()
 
+    def test_recursive_error(self, tmpdir, dataset_df_path):
+        """Check using as a pandas.DataFrame."""
+
+        class TestTask(task.ElementValidationTask):
+            output_columns = {"transformed_morph_path": None}
+
+            @staticmethod
+            def validation_function(row, data_dir):
+                return result.ValidationResult(
+                    is_valid=True, transformed_morph_path="/tmp/test_path"
+                )
+
+        class TestWorkflow(task.ValidationWorkflow):
+            args = ["transformed_dataset.csv", 2]
+
+            @staticmethod
+            def validation_function(df, data_dir, df_path="reduced_df.csv", parents=None):
+                print(df)  # The recursive error was happening here
+
+            def inputs(self):
+                return {TestTask: {}}
+
+        assert luigi.build(
+            [TestWorkflow(dataset_df=dataset_df_path, result_path=str(tmpdir))],
+            local_scheduler=True,
+        )
+
     class TestReport:
         """Test the report generation after workflow run."""
 

tests/test_util.py

@@ -84,6 +84,7 @@ def test_apply_to_df(nb_processes, redirect_stdout):
             }
         ),
         output_columns={"new_data": None, "pid": None},
+        auto_format=True,
     )
 
     res = util.apply_to_df(