Remove IBM GenAI support and moved legacy GenAI metrics to use CrossProviderInferenceEngine

examples/evaluate_llm_as_judge_from_template.py

@@ -22,8 +22,8 @@
 
 # List of metrics to evaluate
 metrics_to_check = [
-    "metrics.llm_as_judge.rating.llama_3_8b_instruct_ibm_genai_template_mt_bench_single_turn",
-    "metrics.llm_as_judge.rating.llama_3_70b_instruct_ibm_genai_template_generic_single_turn",
+    "metrics.llm_as_judge.rating.llama_3_8b_instruct.mt_bench_single_turn",
+    "metrics.llm_as_judge.rating.llama_3_70b_instruct.generic_single_turn",
 ]
 
 for metric_to_check in metrics_to_check:

prepare/cards/tablebench.py

@@ -13,7 +13,7 @@
 card = TaskCard(
     loader=LoadHF(
         path="Multilingual-Multimodal-NLP/TableBench",
-        revision="90593ad8af90f027f6f478b8c4c1981d9f073a83",  # pragma: allowlist secret
+        revision="90593ad",  # pragma: allowlist secret
         data_classification_policy=["public"],
         splits=["test"],
     ),

prepare/cards/tablebench_data_analysis.py

@@ -14,7 +14,7 @@
 card = TaskCard(
     loader=LoadHF(
         path="Multilingual-Multimodal-NLP/TableBench",
-        revision="90593ad8af90f027f6f478b8c4c1981d9f073a83",  # pragma: allowlist secret
+        revision="90593ad",  # pragma: allowlist secret
         data_classification_policy=["public"],
         splits=["test"],
     ),

prepare/cards/tablebench_fact_checking.py

@@ -14,7 +14,7 @@
 card = TaskCard(
     loader=LoadHF(
         path="Multilingual-Multimodal-NLP/TableBench",
-        revision="90593ad8af90f027f6f478b8c4c1981d9f073a83",  # pragma: allowlist secret
+        revision="90593ad",  # pragma: allowlist secret
         data_classification_policy=["public"],
         splits=["test"],
     ),

prepare/cards/tablebench_numerical_reasoning.py

@@ -14,7 +14,7 @@
 card = TaskCard(
     loader=LoadHF(
         path="Multilingual-Multimodal-NLP/TableBench",
-        revision="90593ad8af90f027f6f478b8c4c1981d9f073a83",  # pragma: allowlist secret
+        revision="90593ad",  # pragma: allowlist secret
         data_classification_policy=["public"],
         splits=["test"],
     ),

prepare/metrics/llm_as_judge/pairwise_rating/llama_3_arena_hard_template.py

@@ -2,7 +2,6 @@
 from unitxt.inference import (
     CrossProviderInferenceEngine,
     GenericInferenceEngine,
-    IbmGenAiInferenceEngine,
     WMLInferenceEngine,
 )
 from unitxt.llm_as_judge import LLMAsJudge
@@ -16,7 +15,6 @@
 
 inference_engines = [
     ("ibm_wml", WMLInferenceEngine),
-    ("ibm_genai", IbmGenAiInferenceEngine),
     ("generic_engine", GenericInferenceEngine),
 ]
 

prepare/metrics/llm_as_judge/rating/llama_3_generic_template.py

@@ -0,0 +1,37 @@
+from unitxt import add_to_catalog
+from unitxt.inference import CrossProviderInferenceEngine
+from unitxt.llm_as_judge_from_template import LLMAsJudge
+
+inference_model = CrossProviderInferenceEngine(
+    model="llama-3-70b-instruct", max_tokens=252
+)
+
+metric = LLMAsJudge(
+    inference_model=inference_model,
+    template="templates.response_assessment.rating.generic_single_turn",
+    task="rating.single_turn",
+    format="formats.chat_api",
+    main_score="llama_3_70b_instruct_template_generic_single_turn",
+    prediction_type=str,
+)
+
+add_to_catalog(
+    metric,
+    "metrics.llm_as_judge.rating.llama_3_70b_instruct.generic_single_turn",
+    overwrite=True,
+)
+
+metric = LLMAsJudge(
+    inference_model=inference_model,
+    template="templates.response_assessment.rating.generic_single_turn_with_reference",
+    task="rating.single_turn_with_reference",
+    format="formats.chat_api",
+    single_reference_per_prediction=True,
+    main_score="llama_3_70b_instruct_template_generic_single_turn_with_reference",
+)
+
+add_to_catalog(
+    metric,
+    "metrics.llm_as_judge.rating.llama_3_70b_instruct.generic_single_turn_with_reference",
+    overwrite=True,
+)

prepare/metrics/llm_as_judge/rating/llama_3_ibm_genai_mt_bench_template.py → prepare/metrics/llm_as_judge/rating/llama_3_mt_bench_template.py

@@ -1,32 +1,32 @@
 from unitxt import add_to_catalog
-from unitxt.inference import IbmGenAiInferenceEngine
+from unitxt.inference import CrossProviderInferenceEngine
 from unitxt.llm_as_judge_from_template import LLMAsJudge
 from unitxt.random_utils import get_seed
 
-model_list = ["meta-llama/llama-3-8b-instruct", "meta-llama/llama-3-70b-instruct"]
-format = "formats.llama3_instruct"
+model_list = ["llama-3-70b-instruct", "llama-3-8b-instruct"]
+format = "formats.chat_api"
 template = "templates.response_assessment.rating.mt_bench_single_turn"
 task = "rating.single_turn"
 
+
 for model_id in model_list:
-    inference_model = IbmGenAiInferenceEngine(
-        model_name=model_id, max_new_tokens=252, random_seed=get_seed()
+    inference_model = CrossProviderInferenceEngine(
+        model=model_id, max_tokens=252, seed=get_seed()
     )
-    model_label = model_id.split("/")[1].replace("-", "_").replace(".", ",").lower()
-    model_label = f"{model_label}_ibm_genai"
+    model_label = model_id.replace("-", "_").replace(".", ",").lower()
     template_label = template.split(".")[-1]
     metric_label = f"{model_label}_template_{template_label}"
     metric = LLMAsJudge(
         inference_model=inference_model,
         template=template,
         task=task,
-        format=format,
+        format="formats.chat_api",
         main_score=metric_label,
         prediction_type=str,
     )
 
     add_to_catalog(
         metric,
-        f"metrics.llm_as_judge.rating.{model_label}_template_{template_label}",
+        f"metrics.llm_as_judge.rating.{model_label}.{template_label}",
         overwrite=True,
     )

prepare/metrics/llm_as_judge/rating/llama_3_ibm_genai_table2text_template.py → prepare/metrics/llm_as_judge/rating/llama_3_table2text_template.py

@@ -13,7 +13,6 @@
         model=model_id, max_tokens=252, seed=get_seed()
     )
     model_label = model_id.replace("-", "_").replace(".", ",").lower()
-    model_label = f"{model_label}"
     template_label = template.split(".")[-1]
     metric_label = f"{model_label}_template_{template_label}"
     metric = LLMAsJudge(

src/unitxt/catalog/cards/tablebench.json

@@ -3,7 +3,7 @@
     "loader": {
         "__type__": "load_hf",
         "path": "Multilingual-Multimodal-NLP/TableBench",
-        "revision": "90593ad8af90f027f6f478b8c4c1981d9f073a83",
+        "revision": "90593ad",
         "data_classification_policy": [
             "public"
         ],

src/unitxt/catalog/cards/tablebench_data_analysis.json

@@ -3,7 +3,7 @@
     "loader": {
         "__type__": "load_hf",
         "path": "Multilingual-Multimodal-NLP/TableBench",
-        "revision": "90593ad8af90f027f6f478b8c4c1981d9f073a83",
+        "revision": "90593ad",
         "data_classification_policy": [
             "public"
         ],

src/unitxt/catalog/cards/tablebench_fact_checking.json

@@ -3,7 +3,7 @@
     "loader": {
         "__type__": "load_hf",
         "path": "Multilingual-Multimodal-NLP/TableBench",
-        "revision": "90593ad8af90f027f6f478b8c4c1981d9f073a83",
+        "revision": "90593ad",
         "data_classification_policy": [
             "public"
         ],

src/unitxt/catalog/cards/tablebench_numerical_reasoning.json

@@ -3,7 +3,7 @@
     "loader": {
         "__type__": "load_hf",
         "path": "Multilingual-Multimodal-NLP/TableBench",
-        "revision": "90593ad8af90f027f6f478b8c4c1981d9f073a83",
+        "revision": "90593ad",
         "data_classification_policy": [
             "public"
         ],

src/unitxt/catalog/metrics/llm_as_judge/rating/llama_3_70b_instruct/mt_bench_single_turn.json

@@ -0,0 +1,14 @@
+{
+    "__type__": "llm_as_judge",
+    "inference_model": {
+        "__type__": "cross_provider_inference_engine",
+        "model": "llama-3-70b-instruct",
+        "max_tokens": 252,
+        "seed": 42
+    },
+    "template": "templates.response_assessment.rating.mt_bench_single_turn",
+    "task": "rating.single_turn",
+    "format": "formats.chat_api",
+    "main_score": "llama_3_70b_instruct_template_mt_bench_single_turn",
+    "prediction_type": "str"
+}