From fcb796fdb57819d06c3f240db4f51e5efd424e5a Mon Sep 17 00:00:00 2001 From: supermario_leo Date: Thu, 4 Jun 2026 10:07:37 +0800 Subject: [PATCH 1/2] Preserve rope_scaling when building Eagle3 transformer config The Eagle3 converter built the drafter's LlamaConfig via build_llama_config_rope_kwargs() but only forwarded rope_theta, never rope_scaling. The helper already supports rope_scaling (and the Eagle v1 converter forwards it), so any Eagle3 checkpoint converted for a rope-scaled target (e.g. Llama-3.1/3.3 "llama3" scaling, linear/dynamic NTK, YaRN) silently lost its scaling and fell back to default RoPE, producing incorrect positional encoding at long context. Forward rope_scaling from the Eagle config, mirroring the Eagle v1 converter, and add regression tests covering both the present and absent cases (transformers v4 rope_scaling and v5 rope_parameters layouts). Signed-off-by: supermario_leo --- .../convert/eagle/eagle3_converter.py | 1 + tests/unit/convert/test_eagle3_converter.py | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/speculators/convert/eagle/eagle3_converter.py b/src/speculators/convert/eagle/eagle3_converter.py index 4d03fe94a..5dd62a523 100644 --- a/src/speculators/convert/eagle/eagle3_converter.py +++ b/src/speculators/convert/eagle/eagle3_converter.py @@ -165,6 +165,7 @@ def _create_transformer_config_from_eagle( head_dim=eagle_config.get("head_dim"), **build_llama_config_rope_kwargs( rope_theta=eagle_config.get("rope_theta", 10000.0), + rope_scaling=eagle_config.get("rope_scaling"), ), **build_llama_config_dtype_kwarg(eagle_config.get("torch_dtype")), ) diff --git a/tests/unit/convert/test_eagle3_converter.py b/tests/unit/convert/test_eagle3_converter.py index 5aa4bd38f..38890a704 100644 --- a/tests/unit/convert/test_eagle3_converter.py +++ b/tests/unit/convert/test_eagle3_converter.py @@ -101,6 +101,64 @@ def test_config_max_position_embeddings_logic( else: assert llama_config.rope_theta == 10000.0 + @pytest.mark.sanity + @patch( + "speculators.convert.eagle.eagle3_converter.PretrainedConfig.get_config_dict" + ) + def test_config_preserves_rope_scaling( + self, mock_get_config, sample_eagle3_config, sample_verifier_config + ): + """rope_scaling from the Eagle3 config must be propagated to the + generated transformer config (e.g. Llama-3.1 "llama3" scaling). The + sibling Eagle (v1) converter already forwards it, and dropping it here + would silently disable RoPE scaling for long-context targets.""" + mock_get_config.return_value = (sample_verifier_config, None) + + rope_scaling = { + "rope_type": "llama3", + "factor": 8.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192, + } + eagle_config = {**sample_eagle3_config, "rope_scaling": rope_scaling} + + converter = Eagle3Converter() + llama_config = converter._create_transformer_config_from_eagle( + eagle_config, "meta-llama/Llama-3.1-8B-Instruct" + ) + + if hasattr(llama_config, "rope_parameters"): + # Transformers v5: scaling fields are merged into rope_parameters + assert llama_config.rope_parameters is not None + assert llama_config.rope_parameters.get("rope_type") == "llama3" + assert llama_config.rope_parameters.get("factor") == 8.0 + else: + # Transformers v4: scaling kept under rope_scaling + assert llama_config.rope_scaling is not None + assert llama_config.rope_scaling.get("factor") == 8.0 + + @pytest.mark.sanity + @patch( + "speculators.convert.eagle.eagle3_converter.PretrainedConfig.get_config_dict" + ) + def test_config_no_rope_scaling_when_absent( + self, mock_get_config, sample_eagle3_config, sample_verifier_config + ): + """When the Eagle3 config has no rope_scaling, the generated config must + not introduce any scaling (regression guard for the fix above).""" + mock_get_config.return_value = (sample_verifier_config, None) + + converter = Eagle3Converter() + llama_config = converter._create_transformer_config_from_eagle( + sample_eagle3_config, "meta-llama/Llama-3.1-8B-Instruct" + ) + + if hasattr(llama_config, "rope_parameters"): + assert llama_config.rope_parameters.get("factor") is None + else: + assert llama_config.rope_scaling is None + @pytest.mark.sanity @patch( "speculators.convert.eagle.eagle3_converter.PretrainedConfig.get_config_dict" From e932ab8d6febf7c520eeb2791bde66d139614f86 Mon Sep 17 00:00:00 2001 From: supermario_leo Date: Sat, 13 Jun 2026 19:45:48 +0800 Subject: [PATCH 2/2] Drop the Eagle3 rope_scaling converter tests per review Remove the two regression tests added in fcb796f as suggested in review. The rope_scaling forwarding fix in eagle3_converter.py is unchanged. Signed-off-by: supermario_leo --- tests/unit/convert/test_eagle3_converter.py | 58 --------------------- 1 file changed, 58 deletions(-) diff --git a/tests/unit/convert/test_eagle3_converter.py b/tests/unit/convert/test_eagle3_converter.py index 5333f6941..14f551a65 100644 --- a/tests/unit/convert/test_eagle3_converter.py +++ b/tests/unit/convert/test_eagle3_converter.py @@ -99,64 +99,6 @@ def test_config_max_position_embeddings_logic( else: assert llama_config.rope_theta == 10000.0 - @pytest.mark.sanity - @patch( - "speculators.convert.eagle.eagle3_converter.PretrainedConfig.get_config_dict" - ) - def test_config_preserves_rope_scaling( - self, mock_get_config, sample_eagle3_config, sample_verifier_config - ): - """rope_scaling from the Eagle3 config must be propagated to the - generated transformer config (e.g. Llama-3.1 "llama3" scaling). The - sibling Eagle (v1) converter already forwards it, and dropping it here - would silently disable RoPE scaling for long-context targets.""" - mock_get_config.return_value = (sample_verifier_config, None) - - rope_scaling = { - "rope_type": "llama3", - "factor": 8.0, - "low_freq_factor": 1.0, - "high_freq_factor": 4.0, - "original_max_position_embeddings": 8192, - } - eagle_config = {**sample_eagle3_config, "rope_scaling": rope_scaling} - - converter = Eagle3Converter() - llama_config = converter._create_transformer_config_from_eagle( - eagle_config, "meta-llama/Llama-3.1-8B-Instruct" - ) - - if hasattr(llama_config, "rope_parameters"): - # Transformers v5: scaling fields are merged into rope_parameters - assert llama_config.rope_parameters is not None - assert llama_config.rope_parameters.get("rope_type") == "llama3" - assert llama_config.rope_parameters.get("factor") == 8.0 - else: - # Transformers v4: scaling kept under rope_scaling - assert llama_config.rope_scaling is not None - assert llama_config.rope_scaling.get("factor") == 8.0 - - @pytest.mark.sanity - @patch( - "speculators.convert.eagle.eagle3_converter.PretrainedConfig.get_config_dict" - ) - def test_config_no_rope_scaling_when_absent( - self, mock_get_config, sample_eagle3_config, sample_verifier_config - ): - """When the Eagle3 config has no rope_scaling, the generated config must - not introduce any scaling (regression guard for the fix above).""" - mock_get_config.return_value = (sample_verifier_config, None) - - converter = Eagle3Converter() - llama_config = converter._create_transformer_config_from_eagle( - sample_eagle3_config, "meta-llama/Llama-3.1-8B-Instruct" - ) - - if hasattr(llama_config, "rope_parameters"): - assert llama_config.rope_parameters.get("factor") is None - else: - assert llama_config.rope_scaling is None - @pytest.mark.sanity @patch( "speculators.convert.eagle.eagle3_converter.PretrainedConfig.get_config_dict"