(UI + Proxy) Cache Health Check Page - Cleanup/Improvements (BerriAI#8665)

* fixes for redis cache ping serialization

* fix cache ping check

* fix cache health check ui

* working error details on ui

* ui expand / collapse error

* move cache health check to diff file

* fix displaying error from cache health check

* ui allow copying errors

* ui cache health fixes

* show redis details

* clean up cache health page

* ui polish fixes

* fix error handling on cache health page

* fix redis_cache_params on cache ping response

* error handling

* cache health ping response

* fx error response from cache ping

* parsedLitellmParams

* fix cache health check

* fix cache health page

* cache safely handle json dumps issues

* test caching routes

* test_primitive_types

* fix caching routes

* litellm_mapped_tests

* fix pytest-mock

* fix _serialize

* fix linting on safe dumps

* test_default_max_depth

* pip install "pytest-mock==3.12.0"

* litellm_mapped_tests_coverage

* add readme on new litellm test dir

Author: ishaan-jaff

.circleci/config.yml

@@ -678,6 +678,49 @@ jobs:
           paths:
             - llm_translation_coverage.xml
             - llm_translation_coverage
+  litellm_mapped_tests:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest-mock==3.12.0"
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.21.1"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml litellm_mapped_tests_coverage.xml
+            mv .coverage litellm_mapped_tests_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - litellm_mapped_tests_coverage.xml
+            - litellm_mapped_tests_coverage
   batches_testing:
     docker:
       - image: cimg/python:3.11
@@ -2316,6 +2359,12 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - litellm_mapped_tests:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - batches_testing:
           filters:
             branches:
@@ -2349,6 +2398,7 @@ workflows:
       - upload-coverage:
           requires:
             - llm_translation_testing
+            - litellm_mapped_tests
             - batches_testing
             - litellm_utils_testing
             - pass_through_unit_testing
@@ -2406,6 +2456,7 @@ workflows:
             - load_testing
             - test_bad_database_url
             - llm_translation_testing
+            - litellm_mapped_tests
             - batches_testing
             - litellm_utils_testing
             - pass_through_unit_testing

.pre-commit-config.yaml

@@ -22,7 +22,7 @@ repos:
     rev: 7.0.0  # The version of flake8 to use
     hooks:
     -  id: flake8
-       exclude: ^litellm/tests/|^litellm/proxy/tests/
+       exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/litellm/|^tests/litellm/
        additional_dependencies: [flake8-print]
        files: litellm/.*\.py
     # -  id: flake8

litellm/litellm_core_utils/safe_json_dumps.py

@@ -0,0 +1,49 @@
+import json
+from typing import Any, Union
+
+
+def safe_dumps(data: Any, max_depth: int = 10) -> str:
+    """
+    Recursively serialize data while detecting circular references.
+    If a circular reference is detected then a marker string is returned.
+    """
+
+    def _serialize(obj: Any, seen: set, depth: int) -> Any:
+        # Check for maximum depth.
+        if depth > max_depth:
+            return "MaxDepthExceeded"
+        # Base-case: if it is a primitive, simply return it.
+        if isinstance(obj, (str, int, float, bool, type(None))):
+            return obj
+        # Check for circular reference.
+        if id(obj) in seen:
+            return "CircularReference Detected"
+        seen.add(id(obj))
+        result: Union[dict, list, tuple, set, str]
+        if isinstance(obj, dict):
+            result = {}
+            for k, v in obj.items():
+                result[k] = _serialize(v, seen, depth + 1)
+            seen.remove(id(obj))
+            return result
+        elif isinstance(obj, list):
+            result = [_serialize(item, seen, depth + 1) for item in obj]
+            seen.remove(id(obj))
+            return result
+        elif isinstance(obj, tuple):
+            result = tuple(_serialize(item, seen, depth + 1) for item in obj)
+            seen.remove(id(obj))
+            return result
+        elif isinstance(obj, set):
+            result = sorted([_serialize(item, seen, depth + 1) for item in obj])
+            seen.remove(id(obj))
+            return result
+        else:
+            # Fall back to string conversion for non-serializable objects.
+            try:
+                return str(obj)
+            except Exception:
+                return "Unserializable Object"
+
+    safe_data = _serialize(data, set(), 0)
+    return json.dumps(safe_data, default=str)

litellm/proxy/_experimental/out/onboarding.html

@@ -2000,6 +2000,7 @@ class ProxyErrorTypes(str, enum.Enum):
     bad_request_error = "bad_request_error"
     not_found_error = "not_found_error"
     validation_error = "bad_request_error"
+    cache_ping_error = "cache_ping_error"
 
 
 DB_CONNECTION_ERROR_TYPES = (httpx.ConnectError, httpx.ReadError, httpx.ReadTimeout)

litellm/proxy/_types.py

@@ -5,8 +5,11 @@
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import RedisCache
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
+from litellm.proxy._types import ProxyErrorTypes, ProxyException
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.types.caching import CachePingResponse
 
 masker = SensitiveDataMasker()
 
@@ -18,6 +21,7 @@
 
 @router.get(
     "/ping",
+    response_model=CachePingResponse,
     dependencies=[Depends(user_api_key_auth)],
 )
 async def cache_ping():
@@ -27,27 +31,17 @@ async def cache_ping():
     litellm_cache_params: Dict[str, Any] = {}
     specific_cache_params: Dict[str, Any] = {}
     try:
-
         if litellm.cache is None:
             raise HTTPException(
                 status_code=503, detail="Cache not initialized. litellm.cache is None"
             )
-        litellm_cache_params = {}
-        specific_cache_params = {}
-        for k, v in vars(litellm.cache).items():
-            try:
-                if k == "cache":
-                    continue
-                litellm_cache_params[k] = v
-            except Exception:
-                litellm_cache_params[k] = "<unable to copy or convert>"
-        for k, v in vars(litellm.cache.cache).items():
-            try:
-                specific_cache_params[k] = v
-            except Exception:
-                specific_cache_params[k] = "<unable to copy or convert>"
-        litellm_cache_params = masker.mask_dict(litellm_cache_params)
-        specific_cache_params = masker.mask_dict(specific_cache_params)
+        litellm_cache_params = masker.mask_dict(vars(litellm.cache))
+        # remove field that might reference itself
+        litellm_cache_params.pop("cache", None)
+        specific_cache_params = (
+            masker.mask_dict(vars(litellm.cache.cache)) if litellm.cache else {}
+        )
+
         if litellm.cache.type == "redis":
             # ping the redis cache
             ping_response = await litellm.cache.ping()
@@ -63,24 +57,35 @@ async def cache_ping():
             )
             verbose_proxy_logger.debug("/cache/ping: done with set_cache()")
 
-            return {
-                "status": "healthy",
-                "cache_type": litellm.cache.type,
-                "ping_response": True,
-                "set_cache_response": "success",
-                "litellm_cache_params": litellm_cache_params,
-                "redis_cache_params": specific_cache_params,
-            }
+            return CachePingResponse(
+                status="healthy",
+                cache_type=str(litellm.cache.type),
+                ping_response=True,
+                set_cache_response="success",
+                litellm_cache_params=safe_dumps(litellm_cache_params),
+                redis_cache_params=safe_dumps(specific_cache_params),
+            )
         else:
-            return {
-                "status": "healthy",
-                "cache_type": litellm.cache.type,
-                "litellm_cache_params": litellm_cache_params,
-            }
+            return CachePingResponse(
+                status="healthy",
+                cache_type=str(litellm.cache.type),
+                litellm_cache_params=safe_dumps(litellm_cache_params),
+            )
     except Exception as e:
-        raise HTTPException(
-            status_code=503,
-            detail=f"Service Unhealthy ({str(e)}).Cache parameters: {litellm_cache_params}.specific_cache_params: {specific_cache_params}",
+        import traceback
+
+        traceback.print_exc()
+        error_message = {
+            "message": f"Service Unhealthy ({str(e)})",
+            "litellm_cache_params": safe_dumps(litellm_cache_params),
+            "redis_cache_params": safe_dumps(specific_cache_params),
+            "traceback": traceback.format_exc(),
+        }
+        raise ProxyException(
+            message=safe_dumps(error_message),
+            type=ProxyErrorTypes.cache_ping_error,
+            param="cache_ping",
+            code=503,
         )
 
 

litellm/proxy/caching_routes.py

@@ -11,5 +11,7 @@ general_settings:
 
 
 litellm_settings:
-  callbacks: ["gcs_bucket"]
-
+  cache: true
+  cache_params:
+    type: redis
+    ttl: 600

litellm/proxy/proxy_config.yaml

@@ -1,6 +1,8 @@
 from enum import Enum
 from typing import Literal, Optional, TypedDict
 
+from pydantic import BaseModel
+
 
 class LiteLLMCacheType(str, Enum):
     LOCAL = "local"
@@ -51,3 +53,12 @@ class RedisPipelineIncrementOperation(TypedDict):
         "no-store": Optional[bool],
     },
 )
+
+
+class CachePingResponse(BaseModel):
+    status: str
+    cache_type: str
+    ping_response: Optional[bool] = None
+    set_cache_response: Optional[str] = None
+    litellm_cache_params: Optional[str] = None
+    redis_cache_params: Optional[str] = None

litellm/types/caching.py

@@ -14,6 +14,7 @@
     "strip_field",
     "_transform_prompt",
     "mask_dict",
+    "_serialize",  # we now set a max depth for this
 ]
 
 

tests/code_coverage_tests/recursive_detector.py

@@ -0,0 +1,80 @@
+import json
+import os
+import sys
+
+import pytest
+
+sys.path.insert(
+    0, os.path.abspath("../../..")
+)  # Adds the parent directory to the system path
+
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+
+
+def test_primitive_types():
+    # Test basic primitive types
+    assert safe_dumps("test") == '"test"'
+    assert safe_dumps(123) == "123"
+    assert safe_dumps(3.14) == "3.14"
+    assert safe_dumps(True) == "true"
+    assert safe_dumps(None) == "null"
+
+
+def test_nested_structures():
+    # Test nested dictionaries and lists
+    data = {"name": "test", "numbers": [1, 2, 3], "nested": {"a": 1, "b": 2}}
+    result = json.loads(safe_dumps(data))
+    assert result["name"] == "test"
+    assert result["numbers"] == [1, 2, 3]
+    assert result["nested"] == {"a": 1, "b": 2}
+
+
+def test_circular_reference():
+    # Test circular reference detection
+    d = {}
+    d["self"] = d
+    result = json.loads(safe_dumps(d))
+    assert result["self"] == "CircularReference Detected"
+
+
+def test_max_depth():
+    # Test maximum depth handling
+    deep_dict = {}
+    current = deep_dict
+    for i in range(15):
+        current["deeper"] = {}
+        current = current["deeper"]
+
+    result = json.loads(safe_dumps(deep_dict, max_depth=5))
+    assert "MaxDepthExceeded" in str(result)
+
+
+def test_default_max_depth():
+    # Test that default max depth still prevents infinite recursion
+    deep_dict = {}
+    current = deep_dict
+    for i in range(1000):  # Create a very deep dictionary
+        current["deeper"] = {}
+        current = current["deeper"]
+
+    result = json.loads(safe_dumps(deep_dict))  # No max_depth parameter provided
+    assert "MaxDepthExceeded" in str(result)
+
+
+def test_complex_types():
+    # Test handling of sets and tuples
+    data = {"set": {1, 2, 3}, "tuple": (4, 5, 6)}
+    result = json.loads(safe_dumps(data))
+    assert result["set"] == [1, 2, 3]  # Sets are converted to sorted lists
+    assert result["tuple"] == [4, 5, 6]  # Tuples are converted to lists
+
+
+def test_unserializable_object():
+    # Test handling of unserializable objects
+    class TestClass:
+        def __str__(self):
+            raise Exception("Cannot convert to string")
+
+    obj = TestClass()
+    result = json.loads(safe_dumps(obj))
+    assert result == "Unserializable Object"