Add pyupgrade as a pre-commit hook

.pre-commit-config.yaml

@@ -26,3 +26,10 @@ repos:
         args: ["--maxkb=1024"]
       - id: check-merge-conflict
       - id: debug-statements
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.20.0
+    hooks:
+      - id: pyupgrade
+        name: Enforce Python 3.10+ idioms
+        args: ["--py310-plus"]

docs/scripts/generate_api_summary.py

@@ -60,7 +60,7 @@ def format_nav_section(nav_dict, indent_level=2):
 
 def read_mkdocs_sections(filename: str = "mkdocs.yml"):
     """Read and parse the mkdocs.yml file into sections."""
-    with open(filename, "r") as f:
+    with open(filename) as f:
         lines = f.readlines()
 
     nav_start = -1

dspy/adapters/types/tool.py

@@ -1,6 +1,7 @@
 import asyncio
 import inspect
-from typing import TYPE_CHECKING, Any, Callable, get_origin, get_type_hints
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, get_origin, get_type_hints
 
 import pydantic
 from jsonschema import ValidationError, validate

dspy/clients/embedding.py

@@ -1,4 +1,5 @@
-from typing import Any, Callable
+from collections.abc import Callable
+from typing import Any
 
 import litellm
 import numpy as np

dspy/dsp/utils/dpr.py

@@ -158,7 +158,7 @@ def __init__(self, **kwargs):
             annotators: None or empty set (only tokenizes).
         """
         self._regexp = regex.compile(
-            "(%s)|(%s)" % (self.ALPHA_NUM, self.NON_WS),
+            f"({self.ALPHA_NUM})|({self.NON_WS})",
             flags=regex.IGNORECASE + regex.UNICODE + regex.MULTILINE,
         )
         if len(kwargs.get("annotators", {})) > 0:

dspy/evaluate/evaluate.py

@@ -3,7 +3,8 @@
 import json
 import logging
 import types
-from typing import TYPE_CHECKING, Any, Callable
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     import pandas as pd

dspy/predict/best_of_n.py

@@ -1,4 +1,4 @@
-from typing import Callable
+from collections.abc import Callable
 
 import dspy
 from dspy.predict.predict import Module, Prediction

dspy/predict/code_act.py

@@ -1,6 +1,6 @@
 import inspect
 import logging
-from typing import Callable
+from collections.abc import Callable
 
 import dspy
 from dspy.adapters.types.tool import Tool

dspy/predict/react.py

@@ -1,5 +1,6 @@
 import logging
-from typing import TYPE_CHECKING, Any, Callable, Literal
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, Literal
 
 from litellm import ContextWindowExceededError
 

dspy/predict/refine.py

@@ -1,6 +1,6 @@
 import inspect
 import textwrap
-from typing import Callable
+from collections.abc import Callable
 
 import orjson
 

dspy/retrievers/weaviate_rm.py

@@ -1,4 +1,3 @@
-
 import dspy
 from dspy.dsp.utils import dotdict
 from dspy.primitives.prediction import Prediction

dspy/streaming/streamify.py

@@ -3,8 +3,9 @@
 import logging
 import threading
 from asyncio import iscoroutinefunction
+from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
 from queue import Queue
-from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, Generator
+from typing import TYPE_CHECKING, Any
 
 import litellm
 import orjson

dspy/teleprompt/avatar_optimizer.py

@@ -1,7 +1,7 @@
+from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
 from random import sample
-from typing import Callable
 
 from pydantic import BaseModel
 from tqdm import tqdm

dspy/teleprompt/bettertogether.py

@@ -1,6 +1,6 @@
 import logging
 import random
-from typing import Callable
+from collections.abc import Callable
 
 import dspy
 from dspy.primitives.example import Example

dspy/teleprompt/bootstrap_finetune.py

@@ -1,6 +1,7 @@
 import logging
 from collections import defaultdict
-from typing import Any, Callable
+from collections.abc import Callable
+from typing import Any
 
 import dspy
 from dspy.adapters.base import Adapter

dspy/teleprompt/bootstrap_trace.py

@@ -1,7 +1,8 @@
 import logging
+from collections.abc import Callable
 from dataclasses import dataclass
 from types import MethodType
-from typing import Any, Callable, TypedDict
+from typing import Any, TypedDict
 
 import dspy
 from dspy.evaluate.evaluate import Evaluate

dspy/teleprompt/gepa/gepa_utils.py

@@ -1,6 +1,7 @@
 import logging
 import random
-from typing import Any, Callable, Protocol, TypedDict
+from collections.abc import Callable
+from typing import Any, Protocol, TypedDict
 
 from gepa import EvaluationBatch, GEPAAdapter
 from gepa.core.adapter import ProposalFn

dspy/teleprompt/grpo.py

@@ -1,7 +1,8 @@
 import logging
 import random
 from collections import Counter
-from typing import Any, Callable, Literal
+from collections.abc import Callable
+from typing import Any, Literal
 
 from dspy.adapters.base import Adapter
 from dspy.adapters.chat_adapter import ChatAdapter

dspy/teleprompt/mipro_optimizer_v2.py

@@ -1,7 +1,8 @@
 import logging
 import random
 from collections import defaultdict
-from typing import TYPE_CHECKING, Any, Callable, Literal
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, Literal
 
 import numpy as np
 

dspy/teleprompt/simba.py

@@ -2,7 +2,8 @@
 
 import logging
 import random
-from typing import Any, Callable
+from collections.abc import Callable
+from typing import Any
 
 import numpy as np
 

dspy/teleprompt/simba_utils.py

@@ -1,7 +1,7 @@
 import inspect
 import logging
 import textwrap
-from typing import Callable
+from collections.abc import Callable
 
 import orjson
 

dspy/utils/annotation.py

@@ -1,7 +1,8 @@
 import inspect
 import re
 import types
-from typing import Callable, ParamSpec, TypeVar, overload
+from collections.abc import Callable
+from typing import ParamSpec, TypeVar, overload
 
 P = ParamSpec("P")
 R = TypeVar("R")

dspy/utils/asyncify.py

@@ -1,4 +1,5 @@
-from typing import TYPE_CHECKING, Any, Awaitable, Callable
+from collections.abc import Awaitable, Callable
+from typing import TYPE_CHECKING, Any
 
 import asyncer
 from anyio import CapacityLimiter

dspy/utils/callback.py

@@ -2,8 +2,9 @@
 import inspect
 import logging
 import uuid
+from collections.abc import Callable
 from contextvars import ContextVar
-from typing import Any, Callable
+from typing import Any
 
 import dspy
 

dspy/utils/exceptions.py

@@ -1,4 +1,3 @@
-
 from dspy.signatures.signature import Signature
 
 

dspy/utils/unbatchify.py

@@ -1,8 +1,9 @@
 import queue
 import threading
 import time
+from collections.abc import Callable
 from concurrent.futures import Future
-from typing import Any, Callable
+from typing import Any
 
 
 class Unbatchify:

dspy/utils/usage_tracker.py

@@ -1,8 +1,9 @@
 """Usage tracking utilities for DSPy."""
 
 from collections import defaultdict
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Any, Generator
+from typing import Any
 
 from dspy.dsp.utils.settings import settings
 

tests/primitives/test_base_module.py

@@ -318,8 +318,8 @@ def __call__(self, question: str) -> Prediction:
     assert results[0].get_lm_usage() is not None
     assert results[1].get_lm_usage() is not None
 
-    assert results[0].get_lm_usage().keys() == set(["openai/gpt-4o-mini"])
-    assert results[1].get_lm_usage().keys() == set(["openai/gpt-3.5-turbo"])
+    assert results[0].get_lm_usage().keys() == {"openai/gpt-4o-mini"}
+    assert results[1].get_lm_usage().keys() == {"openai/gpt-3.5-turbo"}
 
 
 @pytest.mark.asyncio

tests/reliability/complex_types/generated/test_many_types_1/program.py

@@ -24,14 +24,14 @@ class ObjectField(BaseModel):
 
 
 class NestedObjectField(BaseModel):
-    tupleField: Tuple[str, float]
+    tupleField: tuple[str, float]
     enumField: EnumField
     datetimeField: datetime
     literalField: LiteralField
 
 
 class ProgramInputs(BaseModel):
-    tupleField: Tuple[str, float]
+    tupleField: tuple[str, float]
     enumField: EnumField
     datetimeField: datetime
     literalField: LiteralField
@@ -76,15 +76,15 @@ class LiteralField(Enum):
 
 
 class ProcessedNestedObjectField(BaseModel):
-    tupleField: Tuple[str, float]
+    tupleField: tuple[str, float]
     enumField: EnumField
     datetimeField: datetime
     literalField: LiteralField
     additionalField: bool
 
 
 class ProgramOutputs(BaseModel):
-    processedTupleField: Tuple[str, float]
+    processedTupleField: tuple[str, float]
     processedEnumField: ProcessedEnumField
     processedDatetimeField: datetime
     processedLiteralField: ProcessedLiteralField

tests/reliability/generate/__init__.py

@@ -13,8 +13,8 @@
 def generate_test_cases(
     dst_path: str,
     num_inputs: int = 1,
-    program_instructions: Optional[str] = None,
-    input_instructions: Optional[str] = None,
+    program_instructions: str | None = None,
+    input_instructions: str | None = None,
 ) -> list[GeneratedTestCase]:
     os.makedirs(dst_path, exist_ok=True)
     if _directory_contains_program(dst_path):

tests/reliability/generate/utils.py

@@ -46,7 +46,7 @@ def wrapper(*args, **kwargs):
 
 
 @_retry(retries=5)
-def generate_test_program(dst_path: str, additional_instructions: Optional[str] = None) -> dspy.Module:
+def generate_test_program(dst_path: str, additional_instructions: str | None = None) -> dspy.Module:
     """
     Generate a DSPy program for a reliability test case and save it to a destination path.
 
@@ -81,7 +81,7 @@ def generate_models(schema: dict[str, Any], class_name: str) -> str:
             _remove_line_from_file(tmp_model_path, "from __future__ import annotations")
             # Remove comments inserted by datamodel-code-generator from the generated model file
             _remove_comments_from_file(tmp_model_path)
-            with open(tmp_model_path, "r") as f:
+            with open(tmp_model_path) as f:
                 return f.read()
 
     def rename_conflicting_fields(
@@ -155,7 +155,7 @@ def generate_test_inputs(
     dst_path: str,
     program_path: str,
     num_inputs: int,
-    additional_instructions: Optional[str] = None,
+    additional_instructions: str | None = None,
 ):
     """
     Generate test inputs for a reliability test case and save them to a destination path.
@@ -226,7 +226,7 @@ def find_max_input_number(directory):
         shutil.copytree(tmp_dir, dst_path, dirs_exist_ok=True)
 
 
-def load_generated_program(path) -> Tuple[dspy.Module, pydantic.BaseModel]:
+def load_generated_program(path) -> tuple[dspy.Module, pydantic.BaseModel]:
     """
     Loads a generated program from the specified file.
 
@@ -283,7 +283,7 @@ def load_generated_cases(dir_path) -> list[GeneratedTestCase]:
             # Load each JSON test input file in the inputs directory
             for input_file in os.listdir(inputs_path):
                 if input_file.endswith(".json"):
-                    with open(os.path.join(inputs_path, input_file), "r") as f:
+                    with open(os.path.join(inputs_path, input_file)) as f:
                         # Best effort to extract a meaningful enclosing directory name
                         # from the test path that can be used as part of the test case name
                         readable_dir_name = os.path.basename(os.path.dirname(os.path.dirname(root)))
@@ -551,7 +551,7 @@ def expand_refs(schema: dict[str, Any], definitions: dict[str, Any]) -> dict[str
     return expand_refs(signature_schema_with_refs, definitions)
 
 
-def _split_schema(schema: dict[str, Any]) -> Tuple[dict[str, Any], dict[str, Any]]:
+def _split_schema(schema: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
     """
     Split a JSON schema into input and output components based on DSPy field types.
 
@@ -664,7 +664,7 @@ def _remove_line_from_file(file_path: str, line_to_remove: str):
         line_to_remove: The line to remove from the file.
     """
     # Read all lines from the file
-    with open(file_path, "r") as file:
+    with open(file_path) as file:
         lines = file.readlines()
 
     # Write all lines back except the one to remove
@@ -682,7 +682,7 @@ def _remove_comments_from_file(file_path: str) -> None:
         file_path: Path to the file where comments should be removed.
     """
     # Read the file contents
-    with open(file_path, "r") as file:
+    with open(file_path) as file:
         lines = file.readlines()
 
     # Filter out lines that start with '#'
@@ -693,7 +693,7 @@ def _remove_comments_from_file(file_path: str) -> None:
         file.writelines(cleaned_lines)
 
 
-def _write_pretty_json(data: dict[str, Any], path: Optional[str] = None) -> Optional[str]:
+def _write_pretty_json(data: dict[str, Any], path: str | None = None) -> str | None:
     """
     Format JSON data with indentation, and write it to a file if specified.
 

tests/reliability/utils.py

@@ -15,7 +15,7 @@
 def assert_program_output_correct(
     program_input: Any,
     program_output: Any,
-    grading_guidelines: Union[str, list[str]],
+    grading_guidelines: str | list[str],
 ):
     """
     With the help of an LLM judge, assert that the specified output of a DSPy program is correct,
@@ -121,7 +121,7 @@ class ReliabilityTestConf(pydantic.BaseModel):
 @lru_cache(maxsize=None)
 def parse_reliability_conf_yaml(conf_file_path: str) -> ReliabilityTestConf:
     try:
-        with open(conf_file_path, "r") as file:
+        with open(conf_file_path) as file:
             conf = yaml.safe_load(file)
 
         model_dict = {}