From 2d7913b38f1fd6f026310daa5189cda4c586b957 Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Thu, 28 May 2026 15:15:35 +0800 Subject: [PATCH 1/2] fix --- src/mcore_bridge/model/gpts/glm4.py | 1 - src/mcore_bridge/model/gpts/qwen3_next.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mcore_bridge/model/gpts/glm4.py b/src/mcore_bridge/model/gpts/glm4.py index 54c16b3..1da91af 100644 --- a/src/mcore_bridge/model/gpts/glm4.py +++ b/src/mcore_bridge/model/gpts/glm4.py @@ -1,7 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from megatron.core.dist_checkpointing.mapping import ShardedStateDict from megatron.core.extensions.transformer_engine import TENorm -from megatron.core.transformer import transformer_layer from megatron.core.transformer.attention import SelfAttention from megatron.core.transformer.mlp import MLP, apply_swiglu_sharded_factory from megatron.core.transformer.spec_utils import build_module diff --git a/src/mcore_bridge/model/gpts/qwen3_next.py b/src/mcore_bridge/model/gpts/qwen3_next.py index 18adf67..ae8c991 100644 --- a/src/mcore_bridge/model/gpts/qwen3_next.py +++ b/src/mcore_bridge/model/gpts/qwen3_next.py @@ -81,6 +81,8 @@ def __init__(self, config: ModelConfig, hidden_size: int, eps: float = 1e-5): self.eps = eps # Initialize weight to zeros (Zero-Centered), matching HuggingFace Qwen3NextRMSNorm self.weight = torch.nn.Parameter(torch.zeros(hidden_size)) + # Mark weight for SP gradient AllReduce across TP domain (consistent with TENorm/MCoreRMSNorm) + setattr(self.weight, 'sequence_parallel', self.config.sequence_parallel) def _norm(self, x): return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) From a44ef9cf22f4249b1dd1dc8678424ae3084da755 Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Thu, 28 May 2026 15:21:18 +0800 Subject: [PATCH 2/2] fix --- src/mcore_bridge/model/gpts/qwen3_next.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcore_bridge/model/gpts/qwen3_next.py b/src/mcore_bridge/model/gpts/qwen3_next.py index ae8c991..0d4e290 100644 --- a/src/mcore_bridge/model/gpts/qwen3_next.py +++ b/src/mcore_bridge/model/gpts/qwen3_next.py @@ -82,7 +82,7 @@ def __init__(self, config: ModelConfig, hidden_size: int, eps: float = 1e-5): # Initialize weight to zeros (Zero-Centered), matching HuggingFace Qwen3NextRMSNorm self.weight = torch.nn.Parameter(torch.zeros(hidden_size)) # Mark weight for SP gradient AllReduce across TP domain (consistent with TENorm/MCoreRMSNorm) - setattr(self.weight, 'sequence_parallel', self.config.sequence_parallel) + setattr(self.weight, 'sequence_parallel', config.sequence_parallel) def _norm(self, x): return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)