Skip to content

Commit a8afcea

Browse files
dafnapensiondafnapension
committed
fixed mmmu by necessary filters
Signed-off-by: dafnapension <[email protected]>
1 parent 7a48aa9 commit a8afcea

31 files changed

+551
-2
lines changed

‎prepare/cards/mmmu.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
from unitxt.blocks import LoadHF, TaskCard
22
from unitxt.catalog import add_to_catalog
33
from unitxt.collections_operators import Filter
4-
from unitxt.operators import ListFieldValues, MapValues
4+
from unitxt.operators import (
5+
FilterByCondition,
6+
FilterByExpression,
7+
ListFieldValues,
8+
MapValues,
9+
)
510
from unitxt.processors import LiteralEval, Lower
611
from unitxt.splitters import RenameSplits
712
from unitxt.string_operators import MapReplace
@@ -40,6 +45,8 @@
4045
"Sociology",
4146
]
4247

48+
mapping = {"A": 0, "B": 1, "C": 2, "D": 3, "E": 4, "?": None}
49+
4350
for name in config_names:
4451
card = TaskCard(
4552
loader=LoadHF(
@@ -60,10 +67,12 @@
6067
),
6168
LiteralEval(field="choices"),
6269
Lower(field="subfield", to_field="topic"),
70+
FilterByCondition(values={"answer": list(mapping.keys())}, condition="in"),
6371
MapValues(
6472
field="answer",
65-
mapping={"A": 0, "B": 1, "C": 2, "D": 3, "E": 4, "?": None},
73+
mapping=mapping,
6674
),
75+
FilterByExpression(expression="answer < len(choices)"),
6776
],
6877
task="tasks.qa.multiple_choice.with_topic",
6978
templates="templates.qa.multiple_choice.with_topic.all",

‎src/unitxt/catalog/cards/mmmu/accounting.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@
6161
"field": "subfield",
6262
"to_field": "topic"
6363
},
64+
{
65+
"__type__": "filter_by_condition",
66+
"values": {
67+
"answer": [
68+
"A",
69+
"B",
70+
"C",
71+
"D",
72+
"E",
73+
"?"
74+
]
75+
},
76+
"condition": "in"
77+
},
6478
{
6579
"__type__": "map_values",
6680
"field": "answer",
@@ -72,6 +86,10 @@
7286
"E": 4,
7387
"?": null
7488
}
89+
},
90+
{
91+
"__type__": "filter_by_expression",
92+
"expression": "answer < len(choices)"
7593
}
7694
],
7795
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/agriculture.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@
6161
"field": "subfield",
6262
"to_field": "topic"
6363
},
64+
{
65+
"__type__": "filter_by_condition",
66+
"values": {
67+
"answer": [
68+
"A",
69+
"B",
70+
"C",
71+
"D",
72+
"E",
73+
"?"
74+
]
75+
},
76+
"condition": "in"
77+
},
6478
{
6579
"__type__": "map_values",
6680
"field": "answer",
@@ -72,6 +86,10 @@
7286
"E": 4,
7387
"?": null
7488
}
89+
},
90+
{
91+
"__type__": "filter_by_expression",
92+
"expression": "answer < len(choices)"
7593
}
7694
],
7795
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/architecture_and_engineering.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@
6161
"field": "subfield",
6262
"to_field": "topic"
6363
},
64+
{
65+
"__type__": "filter_by_condition",
66+
"values": {
67+
"answer": [
68+
"A",
69+
"B",
70+
"C",
71+
"D",
72+
"E",
73+
"?"
74+
]
75+
},
76+
"condition": "in"
77+
},
6478
{
6579
"__type__": "map_values",
6680
"field": "answer",
@@ -72,6 +86,10 @@
7286
"E": 4,
7387
"?": null
7488
}
89+
},
90+
{
91+
"__type__": "filter_by_expression",
92+
"expression": "answer < len(choices)"
7593
}
7694
],
7795
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/art.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@
6161
"field": "subfield",
6262
"to_field": "topic"
6363
},
64+
{
65+
"__type__": "filter_by_condition",
66+
"values": {
67+
"answer": [
68+
"A",
69+
"B",
70+
"C",
71+
"D",
72+
"E",
73+
"?"
74+
]
75+
},
76+
"condition": "in"
77+
},
6478
{
6579
"__type__": "map_values",
6680
"field": "answer",
@@ -72,6 +86,10 @@
7286
"E": 4,
7387
"?": null
7488
}
89+
},
90+
{
91+
"__type__": "filter_by_expression",
92+
"expression": "answer < len(choices)"
7593
}
7694
],
7795
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/art_theory.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@
6161
"field": "subfield",
6262
"to_field": "topic"
6363
},
64+
{
65+
"__type__": "filter_by_condition",
66+
"values": {
67+
"answer": [
68+
"A",
69+
"B",
70+
"C",
71+
"D",
72+
"E",
73+
"?"
74+
]
75+
},
76+
"condition": "in"
77+
},
6478
{
6579
"__type__": "map_values",
6680
"field": "answer",
@@ -72,6 +86,10 @@
7286
"E": 4,
7387
"?": null
7488
}
89+
},
90+
{
91+
"__type__": "filter_by_expression",
92+
"expression": "answer < len(choices)"
7593
}
7694
],
7795
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/basic_medical_science.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@
6161
"field": "subfield",
6262
"to_field": "topic"
6363
},
64+
{
65+
"__type__": "filter_by_condition",
66+
"values": {
67+
"answer": [
68+
"A",
69+
"B",
70+
"C",
71+
"D",
72+
"E",
73+
"?"
74+
]
75+
},
76+
"condition": "in"
77+
},
6478
{
6579
"__type__": "map_values",
6680
"field": "answer",
@@ -72,6 +86,10 @@
7286
"E": 4,
7387
"?": null
7488
}
89+
},
90+
{
91+
"__type__": "filter_by_expression",
92+
"expression": "answer < len(choices)"
7593
}
7694
],
7795
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/biology.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@
6161
"field": "subfield",
6262
"to_field": "topic"
6363
},
64+
{
65+
"__type__": "filter_by_condition",
66+
"values": {
67+
"answer": [
68+
"A",
69+
"B",
70+
"C",
71+
"D",
72+
"E",
73+
"?"
74+
]
75+
},
76+
"condition": "in"
77+
},
6478
{
6579
"__type__": "map_values",
6680
"field": "answer",
@@ -72,6 +86,10 @@
7286
"E": 4,
7387
"?": null
7488
}
89+
},
90+
{
91+
"__type__": "filter_by_expression",
92+
"expression": "answer < len(choices)"
7593
}
7694
],
7795
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/chemistry.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@
6161
"field": "subfield",
6262
"to_field": "topic"
6363
},
64+
{
65+
"__type__": "filter_by_condition",
66+
"values": {
67+
"answer": [
68+
"A",
69+
"B",
70+
"C",
71+
"D",
72+
"E",
73+
"?"
74+
]
75+
},
76+
"condition": "in"
77+
},
6478
{
6579
"__type__": "map_values",
6680
"field": "answer",
@@ -72,6 +86,10 @@
7286
"E": 4,
7387
"?": null
7488
}
89+
},
90+
{
91+
"__type__": "filter_by_expression",
92+
"expression": "answer < len(choices)"
7593
}
7694
],
7795
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/clinical_medicine.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@
6161
"field": "subfield",
6262
"to_field": "topic"
6363
},
64+
{
65+
"__type__": "filter_by_condition",
66+
"values": {
67+
"answer": [
68+
"A",
69+
"B",
70+
"C",
71+
"D",
72+
"E",
73+
"?"
74+
]
75+
},
76+
"condition": "in"
77+
},
6478
{
6579
"__type__": "map_values",
6680
"field": "answer",
@@ -72,6 +86,10 @@
7286
"E": 4,
7387
"?": null
7488
}
89+
},
90+
{
91+
"__type__": "filter_by_expression",
92+
"expression": "answer < len(choices)"
7593
}
7694
],
7795
"task": "tasks.qa.multiple_choice.with_topic",

0 commit comments

Comments
 (0)