cook options when empty rather than filter out

Signed-off-by: dafnapension <[email protected]>

Author: dafnapension

prepare/cards/mmmu.py

@@ -2,8 +2,7 @@
 from unitxt.catalog import add_to_catalog
 from unitxt.collections_operators import Filter
 from unitxt.operators import (
-    FilterByCondition,
-    FilterByExpression,
+    ExecuteExpression,
     ListFieldValues,
     MapValues,
 )
@@ -45,7 +44,7 @@
     "Sociology",
 ]
 
-mapping = {"A": 0, "B": 1, "C": 2, "D": 3, "E": 4, "?": None}
+mapping = {"A": 0, "B": 1, "C": 2, "D": 3, "E": 4, "F": 5, "G": 6, "H": 7, "I": 8}
 
 for name in config_names:
     card = TaskCard(
@@ -58,6 +57,14 @@
                 fields=[f"image_{i}" for i in range(1, 8)], to_field="media/images"
             ),
             Filter(field="media/images", values=[None]),
+            ExecuteExpression(
+                expression="options if options != '[]' else '[\"'+answer+'\"]'",
+                to_field="options",
+            ),
+            ExecuteExpression(
+                expression="'A' if options == '[\"'+answer+'\"]' else answer",
+                to_field="answer",
+            ),
             MapReplace(
                 field_to_field={"question": "question", "options": "choices"},
                 mapping={
@@ -67,12 +74,10 @@
             ),
             LiteralEval(field="choices"),
             Lower(field="subfield", to_field="topic"),
-            FilterByCondition(values={"answer": list(mapping.keys())}, condition="in"),
             MapValues(
                 field="answer",
                 mapping=mapping,
             ),
-            FilterByExpression(expression="answer < len(choices)"),
         ],
         task="tasks.qa.multiple_choice.with_topic",
         templates="templates.qa.multiple_choice.with_topic.all",

src/unitxt/catalog/cards/mmmu/accounting.json

@@ -36,6 +36,16 @@
                 null
             ]
         },
+        {
+            "__type__": "execute_expression",
+            "expression": "options if options != '[]' else '[\"'+answer+'\"]'",
+            "to_field": "options"
+        },
+        {
+            "__type__": "execute_expression",
+            "expression": "'A' if options == '[\"'+answer+'\"]' else answer",
+            "to_field": "answer"
+        },
         {
             "__type__": "map_replace",
             "field_to_field": {
@@ -61,20 +71,6 @@
             "field": "subfield",
             "to_field": "topic"
         },
-        {
-            "__type__": "filter_by_condition",
-            "values": {
-                "answer": [
-                    "A",
-                    "B",
-                    "C",
-                    "D",
-                    "E",
-                    "?"
-                ]
-            },
-            "condition": "in"
-        },
         {
             "__type__": "map_values",
             "field": "answer",
@@ -84,12 +80,11 @@
                 "C": 2,
                 "D": 3,
                 "E": 4,
-                "?": null
+                "F": 5,
+                "G": 6,
+                "H": 7,
+                "I": 8
             }
-        },
-        {
-            "__type__": "filter_by_expression",
-            "expression": "answer < len(choices)"
         }
     ],
     "task": "tasks.qa.multiple_choice.with_topic",

src/unitxt/catalog/cards/mmmu/agriculture.json

@@ -36,6 +36,16 @@
                 null
             ]
         },
+        {
+            "__type__": "execute_expression",
+            "expression": "options if options != '[]' else '[\"'+answer+'\"]'",
+            "to_field": "options"
+        },
+        {
+            "__type__": "execute_expression",
+            "expression": "'A' if options == '[\"'+answer+'\"]' else answer",
+            "to_field": "answer"
+        },
         {
             "__type__": "map_replace",
             "field_to_field": {
@@ -61,20 +71,6 @@
             "field": "subfield",
             "to_field": "topic"
         },
-        {
-            "__type__": "filter_by_condition",
-            "values": {
-                "answer": [
-                    "A",
-                    "B",
-                    "C",
-                    "D",
-                    "E",
-                    "?"
-                ]
-            },
-            "condition": "in"
-        },
         {
             "__type__": "map_values",
             "field": "answer",
@@ -84,12 +80,11 @@
                 "C": 2,
                 "D": 3,
                 "E": 4,
-                "?": null
+                "F": 5,
+                "G": 6,
+                "H": 7,
+                "I": 8
             }
-        },
-        {
-            "__type__": "filter_by_expression",
-            "expression": "answer < len(choices)"
         }
     ],
     "task": "tasks.qa.multiple_choice.with_topic",

src/unitxt/catalog/cards/mmmu/architecture_and_engineering.json

@@ -36,6 +36,16 @@
                 null
             ]
         },
+        {
+            "__type__": "execute_expression",
+            "expression": "options if options != '[]' else '[\"'+answer+'\"]'",
+            "to_field": "options"
+        },
+        {
+            "__type__": "execute_expression",
+            "expression": "'A' if options == '[\"'+answer+'\"]' else answer",
+            "to_field": "answer"
+        },
         {
             "__type__": "map_replace",
             "field_to_field": {
@@ -61,20 +71,6 @@
             "field": "subfield",
             "to_field": "topic"
         },
-        {
-            "__type__": "filter_by_condition",
-            "values": {
-                "answer": [
-                    "A",
-                    "B",
-                    "C",
-                    "D",
-                    "E",
-                    "?"
-                ]
-            },
-            "condition": "in"
-        },
         {
             "__type__": "map_values",
             "field": "answer",
@@ -84,12 +80,11 @@
                 "C": 2,
                 "D": 3,
                 "E": 4,
-                "?": null
+                "F": 5,
+                "G": 6,
+                "H": 7,
+                "I": 8
             }
-        },
-        {
-            "__type__": "filter_by_expression",
-            "expression": "answer < len(choices)"
         }
     ],
     "task": "tasks.qa.multiple_choice.with_topic",

src/unitxt/catalog/cards/mmmu/art.json

@@ -36,6 +36,16 @@
                 null
             ]
         },
+        {
+            "__type__": "execute_expression",
+            "expression": "options if options != '[]' else '[\"'+answer+'\"]'",
+            "to_field": "options"
+        },
+        {
+            "__type__": "execute_expression",
+            "expression": "'A' if options == '[\"'+answer+'\"]' else answer",
+            "to_field": "answer"
+        },
         {
             "__type__": "map_replace",
             "field_to_field": {
@@ -61,20 +71,6 @@
             "field": "subfield",
             "to_field": "topic"
         },
-        {
-            "__type__": "filter_by_condition",
-            "values": {
-                "answer": [
-                    "A",
-                    "B",
-                    "C",
-                    "D",
-                    "E",
-                    "?"
-                ]
-            },
-            "condition": "in"
-        },
         {
             "__type__": "map_values",
             "field": "answer",
@@ -84,12 +80,11 @@
                 "C": 2,
                 "D": 3,
                 "E": 4,
-                "?": null
+                "F": 5,
+                "G": 6,
+                "H": 7,
+                "I": 8
             }
-        },
-        {
-            "__type__": "filter_by_expression",
-            "expression": "answer < len(choices)"
         }
     ],
     "task": "tasks.qa.multiple_choice.with_topic",

src/unitxt/catalog/cards/mmmu/art_theory.json

@@ -36,6 +36,16 @@
                 null
             ]
         },
+        {
+            "__type__": "execute_expression",
+            "expression": "options if options != '[]' else '[\"'+answer+'\"]'",
+            "to_field": "options"
+        },
+        {
+            "__type__": "execute_expression",
+            "expression": "'A' if options == '[\"'+answer+'\"]' else answer",
+            "to_field": "answer"
+        },
         {
             "__type__": "map_replace",
             "field_to_field": {
@@ -61,20 +71,6 @@
             "field": "subfield",
             "to_field": "topic"
         },
-        {
-            "__type__": "filter_by_condition",
-            "values": {
-                "answer": [
-                    "A",
-                    "B",
-                    "C",
-                    "D",
-                    "E",
-                    "?"
-                ]
-            },
-            "condition": "in"
-        },
         {
             "__type__": "map_values",
             "field": "answer",
@@ -84,12 +80,11 @@
                 "C": 2,
                 "D": 3,
                 "E": 4,
-                "?": null
+                "F": 5,
+                "G": 6,
+                "H": 7,
+                "I": 8
             }
-        },
-        {
-            "__type__": "filter_by_expression",
-            "expression": "answer < len(choices)"
         }
     ],
     "task": "tasks.qa.multiple_choice.with_topic",

src/unitxt/catalog/cards/mmmu/basic_medical_science.json

@@ -36,6 +36,16 @@
                 null
             ]
         },
+        {
+            "__type__": "execute_expression",
+            "expression": "options if options != '[]' else '[\"'+answer+'\"]'",
+            "to_field": "options"
+        },
+        {
+            "__type__": "execute_expression",
+            "expression": "'A' if options == '[\"'+answer+'\"]' else answer",
+            "to_field": "answer"
+        },
         {
             "__type__": "map_replace",
             "field_to_field": {
@@ -61,20 +71,6 @@
             "field": "subfield",
             "to_field": "topic"
         },
-        {
-            "__type__": "filter_by_condition",
-            "values": {
-                "answer": [
-                    "A",
-                    "B",
-                    "C",
-                    "D",
-                    "E",
-                    "?"
-                ]
-            },
-            "condition": "in"
-        },
         {
             "__type__": "map_values",
             "field": "answer",
@@ -84,12 +80,11 @@
                 "C": 2,
                 "D": 3,
                 "E": 4,
-                "?": null
+                "F": 5,
+                "G": 6,
+                "H": 7,
+                "I": 8
             }
-        },
-        {
-            "__type__": "filter_by_expression",
-            "expression": "answer < len(choices)"
         }
     ],
     "task": "tasks.qa.multiple_choice.with_topic",