diff --git a/rust/src/chat/src/output/harmony/tests.rs b/rust/src/chat/src/output/harmony/tests.rs index 91cb52fd0db1..cdb272e2cced 100644 --- a/rust/src/chat/src/output/harmony/tests.rs +++ b/rust/src/chat/src/output/harmony/tests.rs @@ -1,14 +1,8 @@ -//! Harmony output tests share the upstream `openai-harmony` tiktoken cache. -//! -//! Use a file lock for tests that load the encoding so `cargo nextest` cannot -//! start multiple processes that concurrently populate the same cache file. - use std::sync::Arc; use futures::executor::block_on; use futures::{TryStreamExt as _, stream}; use openai_harmony::chat::{Message, Role}; -use serial_test::file_serial; use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, Finished}; use super::*; @@ -91,7 +85,6 @@ fn request_with_tools() -> ChatRequest { } #[test] -#[file_serial(harmony_tiktoken_cache)] fn interrupted_final_message_is_preserved() { let tokens = completion_tokens(&[text_message("final", "hello")]); let events = block_on(collect_events( @@ -127,7 +120,6 @@ fn interrupted_final_message_is_preserved() { } #[test] -#[file_serial(harmony_tiktoken_cache)] fn eos_flush_preserves_trailing_replacement_text() { let mut tokens = completion_tokens(&[text_message("final", "Hi")]); tokens.pop(); @@ -153,7 +145,6 @@ fn eos_flush_preserves_trailing_replacement_text() { } #[test] -#[file_serial(harmony_tiktoken_cache)] fn interrupted_analysis_message_is_preserved() { let tokens = completion_tokens(&[text_message("analysis", "think")]); let events = block_on(collect_events( @@ -189,7 +180,6 @@ fn interrupted_analysis_message_is_preserved() { } #[test] -#[file_serial(harmony_tiktoken_cache)] fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() { let tokens = completion_tokens(&[ text_message("commentary", "Let me check."), @@ -217,7 +207,6 @@ fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() { } #[test] -#[file_serial(harmony_tiktoken_cache)] fn multiple_messages_get_newline_separators() { let tokens = completion_tokens(&[ text_message("analysis", "first think"), @@ -249,7 +238,6 @@ fn multiple_messages_get_newline_separators() { } #[test] -#[file_serial(harmony_tiktoken_cache)] fn tool_calls_stream_arguments_and_finish_with_local_id_shape() { let tokens = completion_tokens(&[tool_message( "get_weather", @@ -302,7 +290,6 @@ fn tool_calls_stream_arguments_and_finish_with_local_id_shape() { } #[test] -#[file_serial(harmony_tiktoken_cache)] fn semantic_events_precede_same_update_logprobs() { let tokens = completion_tokens(&[text_message("final", "hello")]); let events = block_on(collect_events( @@ -353,7 +340,6 @@ fn rejects_generic_parser_overrides() { } #[test] -#[file_serial(harmony_tiktoken_cache)] fn allows_auto_auto_only() { validate_harmony_parser_overrides(&ParserSelection::Auto, &ParserSelection::Auto).unwrap(); let _ = HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(); diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 15bd4aca23a7..f2ad815af44b 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -1,8 +1,8 @@ -//! Text-level roundtrip tests for the real chat-template and output-processor pairing. +//! Roundtrip tests for the real chat-template and output-processor pairing. //! //! The invariant under test is that a structured assistant message rendered as history can be //! parsed from the generated assistant completion and then rendered back to the exact same -//! assistant-completion text. +//! assistant completion. use std::pin::Pin; use std::sync::Arc; @@ -18,6 +18,10 @@ use vllm_chat::{ RendererSelection, load_model_backends, }; use vllm_text::{DecodedTextEvent, Finished, Prompt}; +use vllm_tokenizer::Tokenizer; + +const TEXT_COMPLETION_CHUNK_CHARS: usize = 7; +const TOKEN_COMPLETION_CHUNK_TOKENS: usize = 1; /// One model/parser configuration used to run the fixed roundtrip fixtures. #[derive(Clone)] @@ -191,14 +195,28 @@ impl RoundtripCase { sort_json_keys: false, } } + + /// GPT-OSS Harmony token-id renderer and native Harmony output processor. + fn gpt_oss() -> Self { + Self { + model_id: "openai/gpt-oss-20b", + assistant_stop_suffix: "", // not applicable for token-id cases + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + thinking_behavior: ThinkingBehavior::Always { value: true }, + json_fmt: compact_json_fmt(), + sort_json_keys: false, + } + } } macro_rules! roundtrip_tests { - ($($case:ident => [$($fixture:ident),* $(,)?]),+ $(,)?) => { + ($($case:ident => [$($(#[$fixture_attr:meta])* $fixture:ident),* $(,)?]),+ $(,)?) => { paste::paste! { $( $( #[tokio::test] + $(#[$fixture_attr])* #[file_serial([])] async fn []() -> Result<()> { [](RoundtripCase::$case()).await @@ -217,9 +235,9 @@ roundtrip_tests! { glm47 => [reasoning_and_content, tool_call_mix], seed_oss => [reasoning_and_content], step3p5 => [reasoning_and_content], - gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history + gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call } /// Run the fixed reasoning+content fixture for one model/parser case. @@ -421,10 +439,10 @@ struct RoundtripResult { parsed_message: AssistantMessage, /// Assistant-completion suffix cut from rendering the expected assistant as /// history. - closed_completion: String, + closed_completion: Prompt, /// Assistant-completion suffix cut after rendering the parsed assistant /// back as history. - rerendered_closed_completion: String, + rerendered_closed_completion: Prompt, } /// Render, parse, and rerender one assistant turn through the production @@ -436,60 +454,59 @@ async fn run_roundtrip( assistant: AssistantMessage, ) -> Result { let renderer = backends.chat_backend.chat_renderer(); - let (prompt, closed_completion_text) = - render_closed_completion(renderer.as_ref(), request, &assistant)?; - let completion_body = closed_completion_text - .strip_suffix(case.assistant_stop_suffix) - .with_context(|| { - format!( - "closed assistant completion did not end with {:?}: {:?}", - case.assistant_stop_suffix, closed_completion_text - ) - })?; - - let parsed_message = - parse_completion(case, backends, request, &prompt, completion_body).await?; - let (_, rerendered_closed_completion) = - render_closed_completion(renderer.as_ref(), request, &parsed_message)?; + let rendered = render_closed_completion(renderer.as_ref(), request, &assistant)?; + + let parsed_message = parse_completion(case, backends, request, &rendered).await?; + let rerendered = render_closed_completion(renderer.as_ref(), request, &parsed_message)?; Ok(RoundtripResult { parsed_message, - closed_completion: closed_completion_text, - rerendered_closed_completion, + closed_completion: rendered.completion, + rerendered_closed_completion: rerendered.completion, }) } +/// Rendered prompt/completion artifacts at the renderer boundary. +struct RenderedTurn { + prompt: Prompt, + completion: Prompt, +} + /// Render `history` as a production prompt and `history + assistant` as closed /// history, then return the production prompt and assistant-completion suffix. fn render_closed_completion( renderer: &dyn vllm_chat::ChatRenderer, base_request: &ChatRequest, assistant: &AssistantMessage, -) -> Result<(String, String)> { +) -> Result { let mut prompt_request = base_request.clone(); prompt_request.chat_options.generation_prompt_mode = GenerationPromptMode::StartNewAssistant; - let prompt = render_text(renderer, &prompt_request).context("failed to render prompt")?; + let prompt = renderer.render(&prompt_request).context("failed to render prompt")?.prompt; let mut full_request = base_request.clone(); full_request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt; full_request.messages.push(ChatMessage::from(assistant.clone())); - let full = render_text(renderer, &full_request).context("failed to render full prompt")?; - - ensure!( - full.starts_with(&prompt), - "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" - ); - let completion = full[prompt.len()..].to_string(); - - Ok((prompt, completion)) -} + let full = renderer.render(&full_request).context("failed to render full prompt")?.prompt; + + let completion = match (&prompt, full) { + (Prompt::Text(prompt), Prompt::Text(full)) => { + ensure!( + full.starts_with(prompt), + "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" + ); + Prompt::Text(full[prompt.len()..].to_string()) + } + (Prompt::TokenIds(prompt), Prompt::TokenIds(full)) => { + ensure!( + full.starts_with(prompt), + "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" + ); + Prompt::TokenIds(full[prompt.len()..].to_vec()) + } + (prompt, full) => bail!("prompt kind changed between renders: {prompt:?} vs {full:?}"), + }; -/// Render one chat request and require a text prompt. -fn render_text(renderer: &dyn vllm_chat::ChatRenderer, request: &ChatRequest) -> Result { - match renderer.render(request)?.prompt { - Prompt::Text(text) => Ok(text), - other => bail!("roundtrip tests expect text prompts, got {other:?}"), - } + Ok(RenderedTurn { prompt, completion }) } /// Feed one rendered assistant completion body into the real output processor @@ -498,13 +515,15 @@ async fn parse_completion( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, base_request: &ChatRequest, - prompt: &str, - completion_body: &str, + rendered: &RenderedTurn, ) -> Result { let tokenizer = backends.text_backend.tokenizer(); - let prompt_token_ids = tokenizer - .encode(prompt, base_request.add_special_tokens) - .context("failed to encode rendered prompt")?; + let prompt_token_ids = match &rendered.prompt { + Prompt::Text(prompt) => tokenizer + .encode(prompt, base_request.add_special_tokens) + .context("failed to encode rendered prompt")?, + Prompt::TokenIds(token_ids) => token_ids.clone(), + }; let mut request = base_request.clone(); let processor = backends.chat_backend.new_chat_output_processor( @@ -515,7 +534,12 @@ async fn parse_completion( }, )?; - let decoded = decoded_completion_stream(prompt_token_ids, completion_body); + let decoded = decoded_completion_stream( + tokenizer.as_ref(), + prompt_token_ids, + &rendered.completion, + case.assistant_stop_suffix, + )?; let mut events = processor.process(decoded)?; while let Some(event) = events.next().await { @@ -538,16 +562,46 @@ async fn parse_completion( /// split into small chunks to exercise streaming parser state across marker /// and JSON boundaries. fn decoded_completion_stream( + tokenizer: &dyn Tokenizer, prompt_token_ids: Vec, - completion_body: &str, -) -> Pin> + Send>> { - let prompt_token_count = prompt_token_ids.len(); + completion: &Prompt, + assistant_stop_suffix: &str, +) -> Result> + Send>>> { let mut events = vec![DecodedTextEvent::Start { - prompt_token_ids: Arc::from(prompt_token_ids.into_boxed_slice()), + prompt_token_ids: Arc::from(prompt_token_ids.clone().into_boxed_slice()), prompt_logprobs: None, }]; - let chunks = split_by_chars(completion_body, 7); + let chunks = match completion { + Prompt::Text(text) => { + let body = text.strip_suffix(assistant_stop_suffix).with_context(|| { + format!( + "closed assistant completion did not end with {:?}: {:?}", + assistant_stop_suffix, text + ) + })?; + split_by_chars(body, TEXT_COMPLETION_CHUNK_CHARS) + .into_iter() + .map(|delta| DecodedCompletionChunk { + delta, + token_ids: Vec::new(), // unused for text-level roundtrip cases + }) + .collect() + } + Prompt::TokenIds(token_ids) => { + ensure!( + assistant_stop_suffix.is_empty(), + "token-id roundtrip cases do not support text stop suffixes" + ); + incremental_decode_chunks( + tokenizer, + &prompt_token_ids, + token_ids, + TOKEN_COMPLETION_CHUNK_TOKENS, + )? + } + }; + if chunks.is_empty() { events.push({ DecodedTextEvent::TextDelta { @@ -555,11 +609,7 @@ fn decoded_completion_stream( token_ids: Vec::new(), logprobs: None, finished: Some(Finished { - usage: vllm_llm::TokenUsage { - prompt_token_count: 0, - output_token_count: 0, - cached_token_count: 0, - }, + usage: Default::default(), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, }), @@ -569,24 +619,26 @@ fn decoded_completion_stream( let last_index = chunks.len() - 1; for (index, chunk) in chunks.into_iter().enumerate() { let finished = (index == last_index).then(|| Finished { - usage: vllm_llm::TokenUsage { - prompt_token_count, - output_token_count: completion_body.chars().count(), - cached_token_count: 0, - }, + usage: Default::default(), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, }); events.push(DecodedTextEvent::TextDelta { - delta: chunk, - token_ids: Vec::new(), + delta: chunk.delta, + token_ids: chunk.token_ids, logprobs: None, finished, }); } } - stream::iter(events).map(Ok).boxed() + Ok(stream::iter(events).map(Ok).boxed()) +} + +/// One decoded completion chunk fed into the output processor. +struct DecodedCompletionChunk { + delta: String, + token_ids: Vec, } /// Split text into chunks containing at most `chunk_chars` Unicode scalar @@ -612,6 +664,49 @@ fn split_by_chars(text: &str, chunk_chars: usize) -> Vec { chunks } +/// Split token ids into chunks containing at most `chunk_size` ids. +fn split_by_count(token_ids: &[u32], chunk_size: usize) -> Vec> { + token_ids.chunks(chunk_size).map(<[u32]>::to_vec).collect() +} + +/// Decode token ids incrementally using the production tokenizer stream. +fn incremental_decode_chunks( + tokenizer: &dyn Tokenizer, + prompt_token_ids: &[u32], + token_ids: &[u32], + chunk_size: usize, +) -> Result> { + let mut decoder = tokenizer.create_decode_stream(prompt_token_ids, false, 0); + let mut chunks = Vec::new(); + for chunk_token_ids in split_by_count(token_ids, chunk_size) { + let mut delta = String::new(); + for token_id in chunk_token_ids.iter().copied() { + decoder.push_token(token_id)?; + while let Some(chunk) = decoder.next_chunk() { + delta.push_str(&chunk); + } + } + chunks.push(DecodedCompletionChunk { + delta, + token_ids: chunk_token_ids, + }); + } + + let (last_chunk, _) = decoder.flush(None)?; + if let Some(last_chunk) = last_chunk { + if let Some(delta) = chunks.last_mut() { + delta.delta.push_str(&last_chunk); + } else { + chunks.push(DecodedCompletionChunk { + delta: last_chunk, + token_ids: Vec::new(), + }); + } + } + + Ok(chunks) +} + /// Build a chat request fixture with parser-enabling tool-choice semantics. fn roundtrip_request( request_id: impl Into,