diff --git a/examples/maxtext/configs/MI300X/mixtral_8x7B-pretrain.yaml b/examples/maxtext/configs/MI300X/mixtral_8x7B-pretrain.yaml index d983c794e..0d1806150 100644 --- a/examples/maxtext/configs/MI300X/mixtral_8x7B-pretrain.yaml +++ b/examples/maxtext/configs/MI300X/mixtral_8x7B-pretrain.yaml @@ -39,3 +39,4 @@ modules: max_target_length: 4096 per_device_batch_size: 12 remat_policy: "save_dot_with_context_except_mlp" + moe_dispatch_no_expert_sharding: true diff --git a/examples/maxtext/configs/MI355X/mixtral_8x7B-pretrain.yaml b/examples/maxtext/configs/MI355X/mixtral_8x7B-pretrain.yaml index 342f0f454..e0da812df 100644 --- a/examples/maxtext/configs/MI355X/mixtral_8x7B-pretrain.yaml +++ b/examples/maxtext/configs/MI355X/mixtral_8x7B-pretrain.yaml @@ -39,3 +39,4 @@ modules: max_target_length: 4096 per_device_batch_size: 11 remat_policy: "minimal" + moe_dispatch_no_expert_sharding: true