From 84d714610c8634394055db9594bfb25450d0df19 Mon Sep 17 00:00:00 2001 From: semio Date: Wed, 8 Jan 2025 10:53:01 +0800 Subject: [PATCH] the litellm proxy config that worked for openai models --- litellm/batch_prompts.jsonl | 10 +++--- litellm/config.yaml | 26 ++++++---------- litellm/generate_prompts.py | 21 ++++++++----- litellm/test_litellm.py | 61 +++++++++++++++++++------------------ 4 files changed, 60 insertions(+), 58 deletions(-) diff --git a/litellm/batch_prompts.jsonl b/litellm/batch_prompts.jsonl index 8c6dadf..f15b8f4 100644 --- a/litellm/batch_prompts.jsonl +++ b/litellm/batch_prompts.jsonl @@ -1,5 +1,5 @@ -{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "What is the capital of France?"}], "temperature": 0.7, "system_fingerprint": "fp_12345"} -{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "Write a hello world program in Python"}], "temperature": 0.7, "system_fingerprint": "fp_12345"} -{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "Explain what is machine learning in simple terms"}], "temperature": 0.7, "system_fingerprint": "fp_12345"} -{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "What is the difference between SQL and NoSQL databases?"}], "temperature": 0.7, "system_fingerprint": "fp_12345"} -{"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How do I make a chocolate cake?"}], "temperature": 0.7, "system_fingerprint": "fp_12345"} +{"custom_id": "request-0", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "What is the capital of France?"}], "temperature": 0.7}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "Write a hello world program in Python"}], "temperature": 0.7}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "Explain what is machine learning in simple terms"}], "temperature": 0.7}} +{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "What is the difference between SQL and NoSQL databases?"}], "temperature": 0.7}} +{"custom_id": "request-4", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "How do I make a chocolate cake?"}], "temperature": 0.7}} diff --git a/litellm/config.yaml b/litellm/config.yaml index f84462a..1f674cf 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -1,25 +1,17 @@ model_list: - # For response gathering - - model_name: "claude-3-sonnet-20240229" + # Works for ALL Providers and needs the default provider credentials in .env + - model_name: "openai/*" litellm_params: - model: "anthropic/claude-3-sonnet-20240229" - api_key: "os.environ/ANTHROPIC_API_KEY" - batch_size: 20 # Enable batch API with max 20 messages per batch - - # For evaluation models - - model_name: "gpt-4" - litellm_params: - model: "openai/gpt-4" - api_key: "os.environ/OPENAI_API_KEY" - - - model_name: "gemini-pro" - litellm_params: - model: "vertex_ai/gemini-pro" - project_id: "os.environ/VERTEX_PROJECT" - location: "os.environ/VERTEX_LOCATION" + model: "openai/*" + api_key: os.environ/OPENAI_API_KEY + organization: os.environ/OPENAI_ORG_ID general_settings: # Enable detailed logging for debugging debug: true # Set a master key for proxy authentication master_key: "os.environ/LITELLM_MASTER_KEY" + +files_settings: + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY diff --git a/litellm/generate_prompts.py b/litellm/generate_prompts.py index 55738d4..3f856ac 100644 --- a/litellm/generate_prompts.py +++ b/litellm/generate_prompts.py @@ -1,9 +1,9 @@ import json # Static configuration values -MODEL = "openai/gpt-4o-mini" +MODEL = "gpt-4o-mini" TEMPERATURE = 0.7 -SYSTEM_FINGERPRINT = "fp_12345" +# SYSTEM_FINGERPRINT = "fp_12345" # List of prompt contents PROMPT_CONTENTS = [ @@ -17,12 +17,19 @@ # Generate complete prompt objects prompts = [ { - "model": MODEL, - "messages": [{"role": "user", "content": content}], - "temperature": TEMPERATURE, - "system_fingerprint": SYSTEM_FINGERPRINT, + "custom_id": f"request-{i}", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": MODEL, + "messages": [ + # {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": content} + ], + "temperature": TEMPERATURE, + }, } - for content in PROMPT_CONTENTS + for i, content in enumerate(PROMPT_CONTENTS) ] # Create a JSONL file with one prompt per line diff --git a/litellm/test_litellm.py b/litellm/test_litellm.py index 8eb9d57..76d0645 100644 --- a/litellm/test_litellm.py +++ b/litellm/test_litellm.py @@ -1,47 +1,50 @@ +import litellm import os import asyncio -import litellm +import time -async def test_batch_api(): - # Step 1: Create file for batch completion - file_response = await litellm.acreate_file( - file=open("litellm/batch_prompts.jsonl", "rb"), +async def main(): + os.environ["OPENAI_API_KEY"] = "sk-12341234" + litellm.api_base = "http://localhost:4000" + + file_name = "batch_prompts.jsonl" + _current_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(_current_dir, file_name) + file_obj = await litellm.acreate_file( + file=open(file_path, "rb"), purpose="batch", custom_llm_provider="openai", ) - print("File created:", file_response) + print("Response from creating file=", file_obj) - # Step 2: Create batch request - batch_response = await litellm.acreate_batch( + create_batch_response = await litellm.acreate_batch( completion_window="24h", endpoint="/v1/chat/completions", - input_file_id=file_response.id, + input_file_id=file_obj.id, custom_llm_provider="openai", - metadata={"test": "litellm_batch_test"}, + metadata={"custom_id": "test_batch_1"}, ) - print("Batch created:", batch_response) - # Step 3: Retrieve batch status - retrieved_batch = await litellm.aretrieve_batch( - batch_id=batch_response.id, custom_llm_provider="openai" - ) - print("Retrieved batch:", retrieved_batch) + print("response from litellm.create_batch=", create_batch_response) - # Step 4: Get file content - content = await litellm.afile_content( - file_id=file_response.id, custom_llm_provider="openai" - ) - print("File content:", content) + batch_id = create_batch_response.id + # batch_id = "batch_677de66faf988190a417909b0deda9a9" - # Step 5: List batches - batches = litellm.list_batches(custom_llm_provider="openai", limit=10) - print("List of batches:", batches) + while True: + batch_status = await litellm.aretrieve_batch( + batch_id, custom_llm_provider="openai" + ) + if batch_status.output_file_id: + content = await litellm.afile_content( + batch_status.output_file_id, custom_llm_provider="openai" + ) + print(content) + break + else: + time.sleep(100) -if __name__ == "__main__": - # Set LiteLLM proxy URL - os.environ["OPENAI_API_BASE"] = "http://localhost:4000" - os.environ["OPENAI_API_KEY"] = "sk-12341234" # Use your LiteLLM proxy key - asyncio.run(test_batch_api()) +if __name__ == "__main__": + asyncio.run(main())