adds enable thinking to model args

src/lighteval/models/abstract_model.py

@@ -81,6 +81,7 @@ class ModelConfig(BaseModel, extra="forbid"):
 
     generation_parameters: GenerationParameters = GenerationParameters()
     system_prompt: str | None = None
+    enable_thinking: bool | None = None
     cache_dir: str = "~/.cache/huggingface/lighteval"
 
     @classmethod

src/lighteval/models/transformers/transformers_model.py

@@ -229,7 +229,10 @@ def __init__(
             model_size = -1
 
         self.prompt_manager = PromptManager(
-            use_chat_template=self.use_chat_template, tokenizer=self.tokenizer, system_prompt=config.system_prompt
+            use_chat_template=self.use_chat_template,
+            tokenizer=self.tokenizer,
+            system_prompt=config.system_prompt,
+            enable_thinking=config.enable_thinking,
         )
 
         # Initialize cache for tokenization and predictions

src/lighteval/models/vllm/vllm_model.py

@@ -202,7 +202,9 @@ def __init__(
 
         self.pairwise_tokenization = config.pairwise_tokenization
 
-        self.prompt_manager = PromptManager(self.use_chat_template, self.tokenizer, config.system_prompt)
+        self.prompt_manager = PromptManager(
+            self.use_chat_template, self.tokenizer, config.system_prompt, config.enable_thinking
+        )
 
         # Initialize cache for tokenization and predictions
         self._cache = SampleCache(config)

src/lighteval/tasks/prompt_manager.py

@@ -40,8 +40,15 @@
 
 
 class PromptManager:
-    def __init__(self, use_chat_template: bool = False, tokenizer=None, system_prompt: str | None = None):
+    def __init__(
+        self,
+        use_chat_template: bool = False,
+        tokenizer=None,
+        system_prompt: str | None = None,
+        enable_thinking: bool | None = None,
+    ):
         self.use_chat_template = use_chat_template
+        self.enable_thinking = enable_thinking
         self.tokenizer = tokenizer
         self.system_prompt = system_prompt  # System prompt to be used in chat templates
 
@@ -119,10 +126,16 @@ def _prepare_chat_template(self, doc: Doc, tokenize: bool = True) -> str:
         if tokenize:  # for local models
             assert self.tokenizer is not None, "Tokenizer must be set for chat template formatting."
 
+            if self.enable_thinking is not None:
+                tokenizer_kwargs = {"enable_thinking": self.enable_thinking}
+            else:
+                tokenizer_kwargs = {}
+
             return self.tokenizer.apply_chat_template(
                 messages,
                 tokenize=False,
                 add_generation_prompt=True,
+                **tokenizer_kwargs,
             )
 
         else:  # for apis