From 47605c0e515f2daf4240cf6ad5d2acc23b12a399 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 3 Aug 2026 08:35:03 +0200 Subject: [PATCH] common : add env to override n_rs_seq This allows us to manually specify how many recurrent state rollback states we use for speculation. Currently it is set to 0 for all non-draft-model speculators, which can hurt performance a lot. But on the other side you have to be carefull to be conservative with the rollback states, as they baloon quick as they are unquantized f32. --- common/common.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/common.h b/common/common.h index 919c0ea103a4..1fbb49f7c187 100644 --- a/common/common.h +++ b/common/common.h @@ -384,6 +384,10 @@ struct common_params_speculative { } uint32_t need_n_rs_seq() const { + if (const char* env = std::getenv("LLAMA_N_RS_SEQ"); env != nullptr) { + return std::stoi(env); + } + bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; });