Merge pull request instructlab#2316 from alimaredia/display-eval-branch-variant-full-score

feat: add full scores to mt-bench/mmlu-branch output

Author: danmcp

requirements.txt

@@ -6,7 +6,7 @@ datasets>=2.18.0
 gguf>=0.6.0
 GitPython>=3.1.42
 httpx>=0.25.0
-instructlab-eval>=0.2.1
+instructlab-eval>=0.3.1
 instructlab-quantize>=0.1.0
 instructlab-schema>=0.4.0
 instructlab-sdg>=0.3.0

src/instructlab/model/evaluate.py

@@ -139,24 +139,39 @@ def validate_model(model: str, model_arg: str = "--model"):
         raise click.exceptions.Exit(1)
 
 
-def sort_score(pairing: tuple[str, float]) -> float:
+def sort_score(pairing: tuple[str, float, float, float]) -> float:
     """helper func for display_branch_eval_summary
     takes a tuple pairing and returns just the score
     """
     return pairing[1]
 
 
-def display_models(model, base_model) -> None:
+def get_benchmark_max_score(benchmark: Benchmark) -> str:
+    # total score for Benchmark.MT_BENCH_BRANCH or Benchmark.MT_Bench
+    max_score = "10.0"
+    if benchmark in (Benchmark.MMLU_BRANCH, Benchmark.MMLU):
+        max_score = "1.0"
+    return max_score
+
+
+def display_models_and_scores(
+    benchmark, model, base_model, model_score, base_model_score
+) -> None:
     """prints the base_model and model with a header"""
-    print("## BASE MODEL")
-    print(base_model)
-    display_model(model)
+    max_score = get_benchmark_max_score(benchmark)
 
+    base_model_score = round(base_model_score, 2)
+    model_score = round(model_score, 2)
+    print("## BASE MODEL (SCORE)")
+    display_model(base_model, base_model_score, max_score)
+    print("\n## MODEL (SCORE)")
+    display_model(model, model_score, max_score)
 
-def display_model(model) -> None:
+
+def display_model(model, model_score, max_score) -> None:
     """prints the given model with a header"""
-    print("\n## MODEL")
-    print(model)
+    model_score = round(model_score, 2)
+    print(f"{model} ({model_score}/{max_score})")
 
 
 def display_error_rate(error_rate) -> None:
@@ -167,37 +182,49 @@ def display_error_rate(error_rate) -> None:
 
 
 def display_branch_eval_summary(
-    improvements: list[tuple[str, float]],
-    regressions: list[tuple[str, float]],
-    no_changes: list[str],
+    benchmark: Benchmark,
+    improvements: list[tuple[str, float, float, float]],
+    regressions: list[tuple[str, float, float, float]],
+    no_changes: list[tuple[str, float]],
     new=None,
 ):
     """takes in results lists from mt_bench_branch benchmark evaluation
     prints out diff between the branches to the user
     """
+    # total score for MT-BENCH-BRANCH
+    max_score = get_benchmark_max_score(benchmark)
+
     if len(improvements) > 0:
         improvements.sort(key=sort_score, reverse=True)
-        print("\n### IMPROVEMENTS:")
+        print(f"\n### IMPROVEMENTS (0.0 to {max_score}):")
         for index, improvement in enumerate(improvements):
-            task, delta = improvement
-            print(f"{index+1}. {task} (+{delta})")
+            task, delta, base_score, new_score = improvement
+            base_score = round(base_score, 2)
+            new_score = round(new_score, 2)
+            print(f"{index+1}. {task}: {base_score} -> {new_score} (+{delta})")
 
     if len(regressions) > 0:
         regressions.sort(key=sort_score)
-        print("\n### REGRESSIONS:")
+        print(f"\n### REGRESSIONS (0.0 to {max_score}):")
         for index, regression in enumerate(regressions):
-            task, delta = regression
-            print(f"{index+1}. {task} ({delta})")
+            task, delta, base_score, new_score = regression
+            base_score = round(base_score, 2)
+            new_score = round(new_score, 2)
+            print(f"{index+1}. {task}: {base_score} -> {new_score} ({delta})")
 
     if len(no_changes) > 0:
-        print("\n### NO CHANGE:")
-        for index, task in enumerate(no_changes):
-            print(f"{index+1}. {task}")
+        print(f"\n### NO CHANGE (0.0 to {max_score}):")
+        for index, entry in enumerate(no_changes):
+            task, avg_score = entry
+            avg_score = round(avg_score, 2)
+            print(f"{index+1}. {task} ({avg_score})")
 
     if new is not None and len(new) > 0:
-        print("\n### NEW:")
-        for index, qna in enumerate(new):
-            print(f"{index+1}. {qna}")
+        print(f"\n### NEW (0.0 to {max_score}):")
+        for index, entry in enumerate(new):
+            qna, avg_score = entry
+            avg_score = round(avg_score, 2)
+            print(f"{index+1}. {qna} ({avg_score})")
 
 
 def qa_pairs_to_qna_to_avg_scores(qa_pairs: list[dict]) -> dict[str, float]:
@@ -576,13 +603,13 @@ def evaluate(
                 if server is not None:
                     server.shutdown()
 
+            max_score = get_benchmark_max_score(Benchmark.MT_BENCH)
             print("# SKILL EVALUATION REPORT")
-            display_model(model)
-            print("\n### AVERAGE:")
-            print(f"{round(overall_score, 2)} (across {len(qa_pairs)})")
-            print("\n### TURN ONE:")
+            print("\n## MODEL (SCORE)")
+            display_model(model, overall_score, max_score)
+            print(f"\n### TURN ONE (0.0 to {max_score}):")
             print(round(turn_scores[0], 2))
-            print("\n### TURN TWO:")
+            print(f"\n### TURN TWO (0.0 to {max_score}):")
             turn2_score = turn_scores[1]
             if isinstance(turn2_score, float):
                 turn2_score = round(turn2_score, 2)
@@ -659,46 +686,64 @@ def evaluate(
                 for i, evaluator in enumerate(evaluators):
                     branch = branches[i]
                     print(f"Evaluating answers for branch {branch}...")
-                    judgement = evaluator.judge_answers(
+                    overall_score, qa_pairs, error_rate = evaluator.judge_answers(
                         api_base, max_workers=max_workers, serving_gpus=effective_gpus
                     )
-
-                    if len(judgement) == 3:
-                        qa_pairs = judgement[1]
-                        error_rate = judgement[2]
-                    else:
-                        qa_pairs = judgement[0]
-                        error_rate = judgement[1]
-
-                    qa_pairs_and_errors.append((qa_pairs, error_rate))
+                    qa_pairs_and_errors.append((overall_score, qa_pairs, error_rate))
             finally:
                 if server is not None:
                     server.shutdown()
 
-            qa_pairs, error_rate = qa_pairs_and_errors[0]
-            base_qa_pairs, base_error_rate = qa_pairs_and_errors[1]
+            overall_score, qa_pairs, error_rate = qa_pairs_and_errors[0]
+            base_overall_score, base_qa_pairs, base_error_rate = qa_pairs_and_errors[1]
 
             qna_to_avg_scores = qa_pairs_to_qna_to_avg_scores(qa_pairs)
             base_qna_to_avg_scores = qa_pairs_to_qna_to_avg_scores(base_qa_pairs)
 
             print("# SKILL EVALUATION REPORT\n")
-            display_models(model, base_model)
+            display_models_and_scores(
+                Benchmark.MT_BENCH_BRANCH,
+                model,
+                base_model,
+                overall_score,
+                base_overall_score,
+            )
 
             improvements, regressions, no_changes, new_qnas = [], [], [], []
             for qna, avg_score in qna_to_avg_scores.items():
                 base_avg_score = base_qna_to_avg_scores.get(qna)
                 if base_avg_score is not None:
                     if avg_score > base_avg_score:
-                        improvements.append((qna, round(avg_score - base_avg_score, 2)))
+                        improvements.append(
+                            (
+                                qna,
+                                round(avg_score - base_avg_score, 2),
+                                base_avg_score,
+                                avg_score,
+                            )
+                        )
                     elif avg_score == base_avg_score:
-                        no_changes.append(qna)
+                        no_changes.append((qna, avg_score))
                     else:
-                        regressions.append((qna, round(avg_score - base_avg_score, 2)))
+                        regressions.append(
+                            (
+                                qna,
+                                round(avg_score - base_avg_score, 2),
+                                base_avg_score,
+                                avg_score,
+                            )
+                        )
                 else:
-                    new_qnas.append((qna))
+                    new_qnas.append((qna, avg_score))
 
             # display summary of evaluation before exiting
-            display_branch_eval_summary(improvements, regressions, no_changes, new_qnas)
+            display_branch_eval_summary(
+                Benchmark.MT_BENCH_BRANCH,
+                improvements,
+                regressions,
+                no_changes,
+                new_qnas,
+            )
             display_error_rate((error_rate + base_error_rate) / 2)
 
         elif benchmark == Benchmark.MMLU:
@@ -732,12 +777,12 @@ def evaluate(
                 if server is not None:
                     server.shutdown()
 
+            max_score = get_benchmark_max_score(Benchmark.MMLU)
             print("# KNOWLEDGE EVALUATION REPORT")
-            display_model(model)
-            print("\n### AVERAGE:")
-            print(f"{round(overall_score, 2)} (across {len(individual_scores)})\n")
+            print("\n## MODEL (SCORE)")
+            display_model(model, overall_score, max_score)
 
-            print("### SCORES:")
+            print(f"\n### SCORES (0.0 to {max_score}):")
             for task, score in individual_scores.items():
                 s = round(score["score"], 2)
                 print(f"{task} - {s}")
@@ -791,16 +836,13 @@ def evaluate(
             base_individual_scores = individual_scores_list[1]
 
             print("# KNOWLEDGE EVALUATION REPORT\n")
-            display_models(model, base_model)
-
-            print("\n### AVERAGE:")
-            delta = round(overall_score - base_overall_score, 2)
-            if delta >= 0:
-                delta_display = f"+{delta}"
-            else:
-                delta_display = delta
-
-            print(f"{delta_display} (across {len(individual_scores)})")
+            display_models_and_scores(
+                Benchmark.MMLU_BRANCH,
+                model,
+                base_model,
+                overall_score,
+                base_overall_score,
+            )
 
             improvements, regressions, no_changes = [], [], []
             for task, score in individual_scores.items():
@@ -809,14 +851,16 @@ def evaluate(
                 b_s = base_score["score"]
                 d = round(s - b_s, 2)
                 if s > b_s:
-                    improvements.append((task, d))
+                    improvements.append((task, d, b_s, s))
                 elif b_s > s:
-                    regressions.append((task, d))
+                    regressions.append((task, d, b_s, s))
                 else:
-                    no_changes.append(task)
+                    no_changes.append((task, s))
 
             # display summary of evaluation before exiting
-            display_branch_eval_summary(improvements, regressions, no_changes)
+            display_branch_eval_summary(
+                Benchmark.MMLU_BRANCH, improvements, regressions, no_changes
+            )
     except EvalError as ee:
         print(ee.message)
         raise click.exceptions.Exit(1)