Adapt rag granite guardian prepare file and catalog

IBM · Feb 7, 2025 · a5cc196 · a5cc196
1 parent 760575c
commit a5cc196
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 7 deletions.
diff --git a/prepare/metrics/rag_granite_guardian.py b/prepare/metrics/rag_granite_guardian.py
@@ -1,6 +1,7 @@
 from unitxt import add_to_catalog
 from unitxt.metrics import GraniteGuardianWMLMetric, MetricPipeline
 from unitxt.operators import Copy, Set
+from unitxt.string_operators import Join
 
 rag_fields = ["ground_truths", "answer", "contexts", "question"]
 
@@ -24,14 +25,20 @@
     metric = GraniteGuardianWMLMetric(
         main_score=metric_name,
         risk_name=granite_risk_name,
+        user_message_field="question",
+        assistant_message_field="answer",
     )
 
     metric_pipeline = MetricPipeline(
         main_score=metric_name,
         metric=metric,
         preprocess_steps=[
+            Join(
+                field="contexts",
+                by="\n"
+            ),
             Copy(
-                field_to_field={field: f"task_data/{field}" for field in rag_fields},
+                field_to_field={field: f"task_data/{field if field != 'contexts' else 'context'}" for field in rag_fields},
                 not_exist_do_nothing=True,
             ),
             Copy(

diff --git a/src/unitxt/catalog/metrics/rag/granite_guardian_answer_relevance.json b/src/unitxt/catalog/metrics/rag/granite_guardian_answer_relevance.json
@@ -4,15 +4,22 @@
     "metric": {
         "__type__": "granite_guardian_wml_metric",
         "main_score": "granite_guardian_answer_relevance",
-        "risk_name": "answer_relevance"
+        "risk_name": "answer_relevance",
+        "user_message_field": "question",
+        "assistant_message_field": "answer"
     },
     "preprocess_steps": [
+        {
+            "__type__": "join",
+            "field": "contexts",
+            "by": "\n"
+        },
         {
             "__type__": "copy",
             "field_to_field": {
                 "ground_truths": "task_data/ground_truths",
                 "answer": "task_data/answer",
-                "contexts": "task_data/contexts",
+                "contexts": "task_data/context",
                 "question": "task_data/question"
             },
             "not_exist_do_nothing": true

diff --git a/src/unitxt/catalog/metrics/rag/granite_guardian_context_relevance.json b/src/unitxt/catalog/metrics/rag/granite_guardian_context_relevance.json
@@ -4,15 +4,22 @@
     "metric": {
         "__type__": "granite_guardian_wml_metric",
         "main_score": "granite_guardian_context_relevance",
-        "risk_name": "context_relevance"
+        "risk_name": "context_relevance",
+        "user_message_field": "question",
+        "assistant_message_field": "answer"
     },
     "preprocess_steps": [
+        {
+            "__type__": "join",
+            "field": "contexts",
+            "by": "\n"
+        },
         {
             "__type__": "copy",
             "field_to_field": {
                 "ground_truths": "task_data/ground_truths",
                 "answer": "task_data/answer",
-                "contexts": "task_data/contexts",
+                "contexts": "task_data/context",
                 "question": "task_data/question"
             },
             "not_exist_do_nothing": true

diff --git a/src/unitxt/catalog/metrics/rag/granite_guardian_groundedness.json b/src/unitxt/catalog/metrics/rag/granite_guardian_groundedness.json
@@ -4,15 +4,22 @@
     "metric": {
         "__type__": "granite_guardian_wml_metric",
         "main_score": "granite_guardian_groundedness",
-        "risk_name": "groundedness"
+        "risk_name": "groundedness",
+        "user_message_field": "question",
+        "assistant_message_field": "answer"
     },
     "preprocess_steps": [
+        {
+            "__type__": "join",
+            "field": "contexts",
+            "by": "\n"
+        },
         {
             "__type__": "copy",
             "field_to_field": {
                 "ground_truths": "task_data/ground_truths",
                 "answer": "task_data/answer",
-                "contexts": "task_data/contexts",
+                "contexts": "task_data/context",
                 "question": "task_data/question"
             },
             "not_exist_do_nothing": true