Remove IBM GenAI support and moved legacy GenAI metrics to use CrossProviderInferenceEngine

examples/evaluate_ensemble_judge.py

@@ -29,7 +29,7 @@
         reference_fields={},
         prediction_type="str",
         metrics=[
-            "metrics.llm_as_judge.conversation_answer_topicality.ensemble_v1_ibmgenai_judges"
+            "metrics.llm_as_judge.conversation_answer_topicality.ensemble_v1_wml_judges"
         ],
     ),
     templates=TemplatesDict(

examples/evaluate_grounded_ensemble_judge.py

@@ -34,7 +34,7 @@
         reference_fields={},
         prediction_type="str",
         metrics=[
-            "metrics.llm_as_judge.conversation_answer_groundedness.ensemble_v1_ibmgenai_judges"
+            "metrics.llm_as_judge.conversation_answer_groundedness.ensemble_v1_wml_judges"
         ],
     ),
     templates=TemplatesDict(

examples/evaluate_idk_judge.py

@@ -33,9 +33,7 @@
         input_fields={"inquiry": "str"},
         reference_fields={},
         prediction_type="str",
-        metrics=[
-            "metrics.llm_as_judge.conversation_answer_idk.llama3_v1_ibmgenai_judges"
-        ],
+        metrics=["metrics.llm_as_judge.conversation_answer_idk.llama3_v1_wml_judges"],
     ),
     templates=TemplatesDict(
         {

examples/evaluate_llm_as_judge_from_template.py

@@ -22,8 +22,8 @@
 
 # List of metrics to evaluate
 metrics_to_check = [
-    "metrics.llm_as_judge.rating.llama_3_8b_instruct_ibm_genai_template_mt_bench_single_turn",
-    "metrics.llm_as_judge.rating.llama_3_70b_instruct_ibm_genai_template_generic_single_turn",
+    "metrics.llm_as_judge.rating.llama_3_8b_instruct.mt_bench_single_turn",
+    "metrics.llm_as_judge.rating.llama_3_70b_instruct.generic_single_turn",
 ]
 
 for metric_to_check in metrics_to_check:

examples/evaluate_using_metrics_ensemble.py

@@ -11,7 +11,7 @@
 ensemble_metric = MetricsEnsemble(
     metrics=[
         "metrics.llm_as_judge.rating.llama_3_70b_instruct.generic_single_turn",
-        "metrics.llm_as_judge.rating.llama_3_8b_instruct_ibm_genai_template_mt_bench_single_turn",
+        "metrics.llm_as_judge.rating.llama_3_8b_instruct.mt_bench_single_turn",
     ],
     weights=[0.75, 0.25],
 )

prepare/cards/scigen.py

@@ -28,7 +28,7 @@
                 }
             ),
         ],
-        task="tasks.generation.from_pair[metrics=[metrics.llm_as_judge.rating.llama_3_1_70b_instruct_cross_provider_template_table2text_single_turn_with_reference]]",
+        task="tasks.generation.from_pair[metrics=[metrics.llm_as_judge.rating.llama_3_1_70b_instruct.table2text_single_turn_with_reference]]",
         templates=[
             "templates.generation.from_pair.default[postprocessors=[processors.lower_case]]"
         ],

prepare/metrics/llm_as_judge/conversation_groundedness.py

@@ -1,17 +1,13 @@
 import json
 
 from unitxt import add_to_catalog
-from unitxt.inference import (
-    IbmGenAiInferenceEngine,
-    IbmGenAiInferenceEngineParams,
-)
+from unitxt.inference import WMLInferenceEngineGeneration
 from unitxt.llm_as_judge import LLMAsJudge
 from unitxt.metrics import (
     RandomForestMetricsEnsemble,
 )
 
-platform = "ibm_gen_ai"
-gen_params = IbmGenAiInferenceEngineParams(max_new_tokens=256)
+platform = "wml"
 
 config_filepath = "prepare/metrics/llm_as_judge/ensemble_grounded_v1.json"
 
@@ -25,8 +21,8 @@
 
 inference_model_lst = []
 for model_name in model_lst:
-    inference_model = IbmGenAiInferenceEngine(
-        model_name=model_name, parameters=gen_params
+    inference_model = WMLInferenceEngineGeneration(
+        model_name=model_name, max_new_tokens=256
     )
     inference_model_lst.append(inference_model)
 
@@ -60,6 +56,6 @@
 # ensemble_metric.load_weights(config_filepath)
 add_to_catalog(
     ensemble_metric,
-    "metrics.llm_as_judge.conversation_answer_groundedness.ensemble_v1_ibmgenai_judges",
+    "metrics.llm_as_judge.conversation_answer_groundedness.ensemble_v1_wml_judges",
     overwrite=True,
 )

prepare/metrics/llm_as_judge/conversation_idk.py

@@ -1,19 +1,14 @@
 from unitxt import add_to_catalog
-from unitxt.inference import (
-    GenericInferenceEngine,
-    IbmGenAiInferenceEngine,
-    IbmGenAiInferenceEngineParams,
-)
+from unitxt.inference import GenericInferenceEngine, WMLInferenceEngineGeneration
 from unitxt.llm_as_judge import LLMAsJudge
 
 template_name = "templates.response_assessment.judges.idk.v1"
 
 inference_models = {
-    "llama3_v1_ibmgenai": {
+    "llama3_v1_wml": {
         "model_name": "llama370binstruct",
-        "inference_model": IbmGenAiInferenceEngine(
-            model_name="meta-llama/llama-3-70b-instruct",
-            parameters=IbmGenAiInferenceEngineParams(max_new_tokens=256),
+        "inference_model": WMLInferenceEngineGeneration(
+            model_name="meta-llama/llama-3-70b-instruct", max_new_tokens=256
         ),
     },
     "generic_inference_engine": {

prepare/metrics/llm_as_judge/conversation_topicality.py

@@ -1,17 +1,13 @@
 import json
 
 from unitxt import add_to_catalog
-from unitxt.inference import (
-    IbmGenAiInferenceEngine,
-    IbmGenAiInferenceEngineParams,
-)
+from unitxt.inference import WMLInferenceEngineGeneration
 from unitxt.llm_as_judge import LLMAsJudge
 from unitxt.metrics import (
     RandomForestMetricsEnsemble,
 )
 
 platform = "ibm_gen_ai"
-gen_params = IbmGenAiInferenceEngineParams(max_new_tokens=256)
 
 config_filepath = "prepare/metrics/llm_as_judge/ensemble_topicality_v1.json"
 
@@ -25,8 +21,8 @@
 
 inference_model_lst = []
 for model_name in model_lst:
-    inference_model = IbmGenAiInferenceEngine(
-        model_name=model_name, parameters=gen_params
+    inference_model = WMLInferenceEngineGeneration(
+        model_name=model_name, max_new_tokens=256
     )
     inference_model_lst.append(inference_model)
 
@@ -60,6 +56,6 @@
 # ensemble_metric.load_weights(config_filepath)
 add_to_catalog(
     ensemble_metric,
-    "metrics.llm_as_judge.conversation_answer_topicality.ensemble_v1_ibmgenai_judges",
+    "metrics.llm_as_judge.conversation_answer_topicality.ensemble_v1_wml_judges",
     overwrite=True,
 )

prepare/metrics/llm_as_judge/pairwise_rating/llama_3_arena_hard_template.py

@@ -2,7 +2,6 @@
 from unitxt.inference import (
     CrossProviderInferenceEngine,
     GenericInferenceEngine,
-    IbmGenAiInferenceEngine,
     WMLInferenceEngine,
 )
 from unitxt.llm_as_judge import LLMAsJudge
@@ -16,7 +15,6 @@
 
 inference_engines = [
     ("ibm_wml", WMLInferenceEngine),
-    ("ibm_genai", IbmGenAiInferenceEngine),
     ("generic_engine", GenericInferenceEngine),
 ]
 

prepare/metrics/llm_as_judge/rating/llama_3_1_cross_provider_table2text_template.py

@@ -10,9 +10,7 @@
     inference_model = CrossProviderInferenceEngine(
         model=model_id, max_tokens=252, seed=get_seed()
     )
-    model_label = (
-        model_id.replace("-", "_").replace(".", ",").lower() + "_cross_provider"
-    )
+    model_label = model_id.replace("-", "_").replace(".", ",").lower()
     template_label = template.split(".")[-1]
     metric = LLMAsJudge(
         inference_model=inference_model,
@@ -24,6 +22,6 @@
     )
     add_to_catalog(
         metric,
-        f"metrics.llm_as_judge.rating.{model_label}_template_{template_label}",
+        f"metrics.llm_as_judge.rating.{model_label}.{template_label}",
         overwrite=True,
     )

prepare/metrics/llm_as_judge/rating/llama_3_generic_template.py

@@ -0,0 +1,37 @@
+from unitxt import add_to_catalog
+from unitxt.inference import CrossProviderInferenceEngine
+from unitxt.llm_as_judge_from_template import LLMAsJudge
+
+inference_model = CrossProviderInferenceEngine(
+    model="llama-3-70b-instruct", max_tokens=252
+)
+
+metric = LLMAsJudge(
+    inference_model=inference_model,
+    template="templates.response_assessment.rating.generic_single_turn",
+    task="rating.single_turn",
+    format="formats.chat_api",
+    main_score="llama_3_70b_instruct_template_generic_single_turn",
+    prediction_type=str,
+)
+
+add_to_catalog(
+    metric,
+    "metrics.llm_as_judge.rating.llama_3_70b_instruct.generic_single_turn",
+    overwrite=True,
+)
+
+metric = LLMAsJudge(
+    inference_model=inference_model,
+    template="templates.response_assessment.rating.generic_single_turn_with_reference",
+    task="rating.single_turn_with_reference",
+    format="formats.chat_api",
+    single_reference_per_prediction=True,
+    main_score="llama_3_70b_instruct_template_generic_single_turn_with_reference",
+)
+
+add_to_catalog(
+    metric,
+    "metrics.llm_as_judge.rating.llama_3_70b_instruct.generic_single_turn_with_reference",
+    overwrite=True,
+)

prepare/metrics/llm_as_judge/rating/llama_3_ibm_genai_mt_bench_template.py → prepare/metrics/llm_as_judge/rating/llama_3_mt_bench_template.py

@@ -1,32 +1,32 @@
 from unitxt import add_to_catalog
-from unitxt.inference import IbmGenAiInferenceEngine
+from unitxt.inference import CrossProviderInferenceEngine
 from unitxt.llm_as_judge_from_template import LLMAsJudge
 from unitxt.random_utils import get_seed
 
-model_list = ["meta-llama/llama-3-8b-instruct", "meta-llama/llama-3-70b-instruct"]
-format = "formats.llama3_instruct"
+model_list = ["llama-3-70b-instruct", "llama-3-8b-instruct"]
+format = "formats.chat_api"
 template = "templates.response_assessment.rating.mt_bench_single_turn"
 task = "rating.single_turn"
 
+
 for model_id in model_list:
-    inference_model = IbmGenAiInferenceEngine(
-        model_name=model_id, max_new_tokens=252, random_seed=get_seed()
+    inference_model = CrossProviderInferenceEngine(
+        model=model_id, max_tokens=252, seed=get_seed()
     )
-    model_label = model_id.split("/")[1].replace("-", "_").replace(".", ",").lower()
-    model_label = f"{model_label}_ibm_genai"
+    model_label = model_id.replace("-", "_").replace(".", ",").lower()
     template_label = template.split(".")[-1]
     metric_label = f"{model_label}_template_{template_label}"
     metric = LLMAsJudge(
         inference_model=inference_model,
         template=template,
         task=task,
-        format=format,
+        format="formats.chat_api",
         main_score=metric_label,
         prediction_type=str,
     )
 
     add_to_catalog(
         metric,
-        f"metrics.llm_as_judge.rating.{model_label}_template_{template_label}",
+        f"metrics.llm_as_judge.rating.{model_label}.{template_label}",
         overwrite=True,
     )

prepare/metrics/llm_as_judge/rating/llama_3_ibm_genai_table2text_template.py → prepare/metrics/llm_as_judge/rating/llama_3_table2text_template.py

@@ -7,13 +7,11 @@
 format = "formats.llama3_instruct"
 template = "templates.response_assessment.rating.table2text_single_turn_with_reference"
 task = "rating.single_turn_with_reference"
-
 for model_id in model_list:
     inference_model = CrossProviderInferenceEngine(
         model=model_id, max_tokens=252, seed=get_seed()
     )
     model_label = model_id.replace("-", "_").replace(".", ",").lower()
-    model_label = f"{model_label}"
     template_label = template.split(".")[-1]
     metric_label = f"{model_label}_template_{template_label}"
     metric = LLMAsJudge(
@@ -27,6 +25,6 @@
 
     add_to_catalog(
         metric,
-        f"metrics.llm_as_judge.rating.{model_label}_template_{template_label}",
+        f"metrics.llm_as_judge.rating.{model_label}.{template_label}",
         overwrite=True,
     )