Ablation Tooling and Direct Model Provider API Keys

ablation_studies/20251207_155002/ablation_metadata.json

@@ -0,0 +1,24 @@
+{
+    "timestamp": "20251207_155002",
+    "mode": "distraction",
+    "count": 10,
+    "tasks_per_server": 15,
+    "description": "Ablation study with distraction mode, count=10",
+    "results": {
+        "single_server": {
+            "success": true,
+            "file": "ablation_single_server_tasks.json",
+            "runner_format": "ablation_single_server_tasks_runner_format.json"
+        },
+        "two_server": {
+            "success": true,
+            "file": "ablation_2server_tasks.json",
+            "runner_format": "ablation_2server_tasks_runner_format.json"
+        },
+        "three_server": {
+            "success": true,
+            "file": "ablation_3server_tasks.json",
+            "runner_format": "ablation_3server_tasks_runner_format.json"
+        }
+    }
+}

ablation_studies/20251207_155002/benchmark_results_20251215_090111/2server_results.json

@@ -0,0 +1,23 @@
+{
+  "task_completion_score": 6.098666666666671,
+  "tool_selection_score": 7.361333333333338,
+  "planning_effectiveness_and_efficiency_score": 4.559999999999999,
+  "task_fulfillment": 5.142222222222225,
+  "grounding": 7.055111111111112,
+  "tool_appropriateness": 7.0968888888888895,
+  "parameter_accuracy": 7.625777777777773,
+  "dependency_awareness": 5.549333333333332,
+  "parallelism_and_efficiency": 3.5706666666666655,
+  "input_schema_compliance": 0.9919478100148106,
+  "valid_tool_name_rate": 0.9975778773047681,
+  "tool_call_success_rate": 0.9378065978494261,
+  "avg_execution_time": 164.02752392027114,
+  "avg_agent_execution_time": 121.83732213338216,
+  "avg_evaluation_time": 113.33587040477329,
+  "task_success_rate": 1.0,
+  "avg_total_rounds": 5.182222222222222,
+  "avg_tool_calls_per_task": 15.528888888888888,
+  "avg_output_tokens": 2308.702222222222,
+  "avg_prompt_tokens": 148705.41777777777,
+  "avg_total_tokens": 151014.12
+}

ablation_studies/20251207_155002/benchmark_results_20251215_090111/3server_results.json

@@ -0,0 +1,23 @@
+{
+  "task_completion_score": 6.2162962962962975,
+  "tool_selection_score": 6.749629629629626,
+  "planning_effectiveness_and_efficiency_score": 4.6577777777777785,
+  "task_fulfillment": 5.389629629629633,
+  "grounding": 7.042962962962963,
+  "tool_appropriateness": 6.482962962962961,
+  "parameter_accuracy": 7.016296296296297,
+  "dependency_awareness": 5.564444444444445,
+  "parallelism_and_efficiency": 3.75111111111111,
+  "input_schema_compliance": 0.9973582605161553,
+  "valid_tool_name_rate": 0.9943669069673736,
+  "tool_call_success_rate": 0.9778455389574375,
+  "avg_execution_time": 191.51877691127635,
+  "avg_agent_execution_time": 130.25742151825517,
+  "avg_evaluation_time": 108.03205980901365,
+  "task_success_rate": 1.0,
+  "avg_total_rounds": 5.014814814814815,
+  "avg_tool_calls_per_task": 14.251851851851852,
+  "avg_output_tokens": 2275.4222222222224,
+  "avg_prompt_tokens": 157254.11111111112,
+  "avg_total_tokens": 159529.53333333333
+}

ablation_studies/20251207_155002/benchmark_results_20251215_090111/benchmark_summary.json

@@ -0,0 +1,18 @@
+{
+  "timestamp": "20251215_090111",
+  "study": "20251207_155002",
+  "models": [
+    "gpt-4o"
+  ],
+  "configs_tested": [
+    "single",
+    "2server",
+    "3server"
+  ],
+  "output_directory": "/home/himaneeshsompalle/mcp-bench-main-3/ablation_studies/20251207_155002/benchmark_results_20251215_090111",
+  "results": {
+    "single": "success",
+    "2server": "success",
+    "3server": "success"
+  }
+}

ablation_studies/20251207_155002/benchmark_results_20251215_090111/single_results.json

@@ -0,0 +1,23 @@
+{
+  "task_completion_score": 6.011794871794872,
+  "tool_selection_score": 7.376923076923077,
+  "planning_effectiveness_and_efficiency_score": 4.603076923076925,
+  "task_fulfillment": 4.990256410256411,
+  "grounding": 7.033333333333336,
+  "tool_appropriateness": 7.133846153846151,
+  "parameter_accuracy": 7.619999999999994,
+  "dependency_awareness": 5.703076923076924,
+  "parallelism_and_efficiency": 3.503076923076923,
+  "input_schema_compliance": 0.9994597301286107,
+  "valid_tool_name_rate": 0.993613337455078,
+  "tool_call_success_rate": 0.9757985890850466,
+  "avg_execution_time": 122.73608196270771,
+  "avg_agent_execution_time": 83.90191582594163,
+  "avg_evaluation_time": 112.25089816924853,
+  "task_success_rate": 1.0,
+  "avg_total_rounds": 3.9256410256410255,
+  "avg_tool_calls_per_task": 11.235897435897435,
+  "avg_output_tokens": 1867.3307692307692,
+  "avg_prompt_tokens": 109814.51538461538,
+  "avg_total_tokens": 111681.84615384616
+}

ablation_studies/20251208_112959/ablation_metadata.json

@@ -0,0 +1,24 @@
+{
+    "timestamp": "20251208_112959",
+    "mode": "distraction",
+    "count": 14,
+    "tasks_per_server": 15,
+    "description": "Ablation study with distraction mode, count=14",
+    "results": {
+        "single_server": {
+            "success": true,
+            "file": "ablation_single_server_tasks.json",
+            "runner_format": "ablation_single_server_tasks_runner_format.json"
+        },
+        "two_server": {
+            "success": true,
+            "file": "ablation_2server_tasks.json",
+            "runner_format": "ablation_2server_tasks_runner_format.json"
+        },
+        "three_server": {
+            "success": true,
+            "file": "ablation_3server_tasks.json",
+            "runner_format": "ablation_3server_tasks_runner_format.json"
+        }
+    }
+}