the litellm proxy config that worked for openai models

litellm/batch_prompts.jsonl

@@ -1,5 +1,5 @@
-{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "What is the capital of France?"}], "temperature": 0.7, "system_fingerprint": "fp_12345"}
-{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "Write a hello world program in Python"}], "temperature": 0.7, "system_fingerprint": "fp_12345"}
-{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "Explain what is machine learning in simple terms"}], "temperature": 0.7, "system_fingerprint": "fp_12345"}
-{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "What is the difference between SQL and NoSQL databases?"}], "temperature": 0.7, "system_fingerprint": "fp_12345"}
-{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How do I make a chocolate cake?"}], "temperature": 0.7, "system_fingerprint": "fp_12345"}
+{"custom_id": "request-0", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "What is the capital of France?"}], "temperature": 0.7}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "Write a hello world program in Python"}], "temperature": 0.7}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "Explain what is machine learning in simple terms"}], "temperature": 0.7}}
+{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "What is the difference between SQL and NoSQL databases?"}], "temperature": 0.7}}
+{"custom_id": "request-4", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "How do I make a chocolate cake?"}], "temperature": 0.7}}

litellm/config.yaml

@@ -1,25 +1,17 @@
 model_list:
-  # For response gathering
-  - model_name: "claude-3-sonnet-20240229"
+  # Works for ALL Providers and needs the default provider credentials in .env
+  - model_name: "openai/*"
     litellm_params:
-      model: "anthropic/claude-3-sonnet-20240229"
-      api_key: "os.environ/ANTHROPIC_API_KEY"
-      batch_size: 20  # Enable batch API with max 20 messages per batch
-
-  # For evaluation models
-  - model_name: "gpt-4"
-    litellm_params:
-      model: "openai/gpt-4"
-      api_key: "os.environ/OPENAI_API_KEY"
-
-  - model_name: "gemini-pro"
-    litellm_params:
-      model: "vertex_ai/gemini-pro"
-      project_id: "os.environ/VERTEX_PROJECT"
-      location: "os.environ/VERTEX_LOCATION"
+      model: "openai/*"
+      api_key: os.environ/OPENAI_API_KEY
+      organization: os.environ/OPENAI_ORG_ID
 
 general_settings:
   # Enable detailed logging for debugging
   debug: true
   # Set a master key for proxy authentication
   master_key: "os.environ/LITELLM_MASTER_KEY"
+
+files_settings:
+  - custom_llm_provider: openai
+    api_key: os.environ/OPENAI_API_KEY

litellm/generate_prompts.py

@@ -1,9 +1,9 @@
 import json
 
 # Static configuration values
-MODEL = "openai/gpt-4o-mini"
+MODEL = "gpt-4o-mini"
 TEMPERATURE = 0.7
-SYSTEM_FINGERPRINT = "fp_12345"
+# SYSTEM_FINGERPRINT = "fp_12345"
 
 # List of prompt contents
 PROMPT_CONTENTS = [
@@ -17,12 +17,19 @@
 # Generate complete prompt objects
 prompts = [
     {
-        "model": MODEL,
-        "messages": [{"role": "user", "content": content}],
-        "temperature": TEMPERATURE,
-        "system_fingerprint": SYSTEM_FINGERPRINT,
+        "custom_id": f"request-{i}",
+        "method": "POST",
+        "url": "/v1/chat/completions",
+        "body": {
+            "model": MODEL,
+            "messages": [
+                # {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": content}
+            ],
+            "temperature": TEMPERATURE,
+        },
     }
-    for content in PROMPT_CONTENTS
+    for i, content in enumerate(PROMPT_CONTENTS)
 ]
 
 # Create a JSONL file with one prompt per line

litellm/test_litellm.py

@@ -1,47 +1,50 @@
+import litellm
 import os
 import asyncio
-import litellm
+import time
 
 
-async def test_batch_api():
-    # Step 1: Create file for batch completion
-    file_response = await litellm.acreate_file(
-        file=open("litellm/batch_prompts.jsonl", "rb"),
+async def main():
+    os.environ["OPENAI_API_KEY"] = "sk-12341234"
+    litellm.api_base = "http://localhost:4000"
+
+    file_name = "batch_prompts.jsonl"
+    _current_dir = os.path.dirname(os.path.abspath(__file__))
+    file_path = os.path.join(_current_dir, file_name)
+    file_obj = await litellm.acreate_file(
+        file=open(file_path, "rb"),
         purpose="batch",
         custom_llm_provider="openai",
     )
-    print("File created:", file_response)
+    print("Response from creating file=", file_obj)
 
-    # Step 2: Create batch request
-    batch_response = await litellm.acreate_batch(
+    create_batch_response = await litellm.acreate_batch(
         completion_window="24h",
         endpoint="/v1/chat/completions",
-        input_file_id=file_response.id,
+        input_file_id=file_obj.id,
         custom_llm_provider="openai",
-        metadata={"test": "litellm_batch_test"},
+        metadata={"custom_id": "test_batch_1"},
     )
-    print("Batch created:", batch_response)
 
-    # Step 3: Retrieve batch status
-    retrieved_batch = await litellm.aretrieve_batch(
-        batch_id=batch_response.id, custom_llm_provider="openai"
-    )
-    print("Retrieved batch:", retrieved_batch)
+    print("response from litellm.create_batch=", create_batch_response)
 
-    # Step 4: Get file content
-    content = await litellm.afile_content(
-        file_id=file_response.id, custom_llm_provider="openai"
-    )
-    print("File content:", content)
+    batch_id = create_batch_response.id
+    # batch_id = "batch_677de66faf988190a417909b0deda9a9"
 
-    # Step 5: List batches
-    batches = litellm.list_batches(custom_llm_provider="openai", limit=10)
-    print("List of batches:", batches)
+    while True:
+        batch_status = await litellm.aretrieve_batch(
+            batch_id, custom_llm_provider="openai"
+        )
 
+        if batch_status.output_file_id:
+            content = await litellm.afile_content(
+                batch_status.output_file_id, custom_llm_provider="openai"
+            )
+            print(content)
+            break
+        else:
+            time.sleep(100)
 
-if __name__ == "__main__":
-    # Set LiteLLM proxy URL
-    os.environ["OPENAI_API_BASE"] = "http://localhost:4000"
-    os.environ["OPENAI_API_KEY"] = "sk-12341234"  # Use your LiteLLM proxy key
 
-    asyncio.run(test_batch_api())
+if __name__ == "__main__":
+    asyncio.run(main())