From cccdc6ecf30d6adcc48315bfb3203d83cd10f997 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Wed, 22 Jul 2026 17:36:05 +0800 Subject: [PATCH 01/11] update --- docs/source/quickstart/rl-training.md | 194 ++++++++++++- examples/agent_train/train_qwen3_moe.sh | 56 ++-- .../training/task_config_claude_code.yaml | 25 ++ .../training/task_config_react.yaml | 45 +++ .../quickstart/training/train_qwen3_moe.sh | 261 ++++++++++++++++++ 5 files changed, 551 insertions(+), 30 deletions(-) create mode 100644 examples/quickstart/training/task_config_claude_code.yaml create mode 100644 examples/quickstart/training/task_config_react.yaml create mode 100644 examples/quickstart/training/train_qwen3_moe.sh diff --git a/docs/source/quickstart/rl-training.md b/docs/source/quickstart/rl-training.md index 398abe94..1062ec2d 100644 --- a/docs/source/quickstart/rl-training.md +++ b/docs/source/quickstart/rl-training.md @@ -1,15 +1,195 @@ # Train an Agent with RL - +This guide demonstrates Agentic RL training for both white-box and black-box agents: -## Before You Start +1. Train `Qwen3-Coder-30B-A3B-Instruct` with the white-box `ReAct Agent`. +2. Train `Qwen3.5-9B` with the black-box `Claude Code` Agent. -## Choose a Training Recipe +## Prerequisites -## Prepare the Task and Dataset +We recommend completing the preceding Quickstart guides before starting training to ensure that the Task dependencies and Sandbox service are working correctly. -## Configure Rollouts and Training +## Prepare the Data -## Launch Training +Both examples train on SWE-reBench and validate on SWE-Bench Verified. The preprocessors convert each dataset row into the Task Config format consumed by Uni-Agent. -## Monitor and Verify the Run +### Training Dataset + +!!! note "Ready-to-use SWE-reBench dataset" + You can directly use our processed `swe-rebench-filtered-1150` dataset, which contains 1,150 training samples. We preprocess and filter the original SWE-reBench examples to make them better suited for Agent RL training. + + **Dataset:** [https://huggingface.co/datasets/dyyyyyyyy/swe-rebench-filtered-1150](https://huggingface.co/datasets/dyyyyyyyy/swe-rebench-filtered-1150) + +Prepare the filtered SWE-reBench split: + +```bash +python3 -m uni_agent.tasks.swe_rebench.preprocess --local-save-dir ~/data/uni_agent +``` + +The command writes: `~/data/uni_agent/swe_rebench_filtered.parquet` + +### Validation Dataset + +Prepare SWE-Bench Verified: + +```bash +python3 -m uni_agent.tasks.swe_bench.preprocess --local-save-dir ~/data/uni_agent +``` + +The command writes: `~/data/uni_agent/swe_bench_verified.parquet` + +The processed rows remain independent of the runtime Sandbox provider. Each row contains the rendered prompt, task metadata, canonical image reference, and per-sample Task Config. + +## Configuration + +### Task Configuration + +The Quickstart provides separate configs for the two Agent types: + +=== "ReAct" + + ```yaml + - name: swe_bench + sandbox: + provider: vefaas + runtime_timeout: 7200 + agent: + name: react + max_steps: 200 + tools: + - name: stateful_shell + command_timeout: 120 + env_vars: + PAGER: "cat" + GIT_PAGER: "cat" + MANPAGER: "cat" + TQDM_DISABLE: "1" + PIP_PROGRESS_BAR: "off" + - name: str_replace_editor + - name: submit + model: + temperature: 1.0 + top_p: 1.0 + max_total_tokens: 131072 + + - name: swe_rebench + sandbox: + provider: vefaas + runtime_timeout: 7200 + agent: + name: react + max_steps: 200 + tools: + - name: stateful_shell + command_timeout: 120 + env_vars: + PAGER: "cat" + GIT_PAGER: "cat" + MANPAGER: "cat" + TQDM_DISABLE: "1" + PIP_PROGRESS_BAR: "off" + - name: str_replace_editor + - name: submit + model: + temperature: 1.0 + top_p: 1.0 + max_total_tokens: 131072 + ``` + +=== "Claude Code" + + ```yaml + - name: swe_bench + sandbox: + provider: vefaas + runtime_timeout: 7200 + agent: + name: claude_code + max_turns: 100 + run_timeout: 7200 + model: + temperature: 1.0 + top_p: 1.0 + max_total_tokens: 131072 + + - name: swe_rebench + sandbox: + provider: vefaas + runtime_timeout: 7200 + agent: + name: claude_code + max_turns: 100 + run_timeout: 7200 + model: + temperature: 1.0 + top_p: 1.0 + max_total_tokens: 131072 + ``` + + !!! warning "Network connectivity" + The Claude Code sandbox must be able to reach the GPU machine hosting its session-scoped Gateway endpoint. + +### Ray Runtime Environment + +Training runs as a Ray job. Use a Runtime Environment to distribute the repository, expose the bundled `verl` source, install lightweight Task and Sandbox dependencies, and pass credentials to every Agent runner. + +=== "veFaaS" + + ```yaml + working_dir: ./ + excludes: ["/.git/"] + + pip: + packages: + - "volcengine-python-sdk" + - "swe-rex" + - "swebench" + + env_vars: + PYTHONPATH: "verl" + PYTHONNOUSERSITE: "1" + TORCH_NCCL_AVOID_RECORD_STREAMS: "1" + CUDA_DEVICE_MAX_CONNECTIONS: "1" + + VEFAAS_FUNCTION_ID: "" + VEFAAS_FUNCTION_ROUTE: "" + VOLCE_ACCESS_KEY: "" + VOLCE_SECRET_KEY: "" + ``` + +=== "Modal" + + ```yaml + working_dir: ./ + excludes: ["/.git/"] + + pip: + packages: + - "modal" + - "swebench" + + env_vars: + PYTHONPATH: "verl" + PYTHONNOUSERSITE: "1" + TORCH_NCCL_AVOID_RECORD_STREAMS: "1" + CUDA_DEVICE_MAX_CONNECTIONS: "1" + + MODAL_TOKEN_ID: "" + MODAL_TOKEN_SECRET: "" + ``` + +## Case 1: ReAct Agent RL + +### Launch Training + +### Monitor the Run + +### Results + +## Case 2: Claude Code RL + +### Launch Training + +### Monitor the Run + +### Results diff --git a/examples/agent_train/train_qwen3_moe.sh b/examples/agent_train/train_qwen3_moe.sh index 6155f223..683c8bf7 100644 --- a/examples/agent_train/train_qwen3_moe.sh +++ b/examples/agent_train/train_qwen3_moe.sh @@ -1,23 +1,22 @@ #!/usr/bin/env bash set -xeuo pipefail -RAY_DATA_HOME=/mnt/hdfs/yyding +DATA_DIR=/mnt/hdfs/yyding +RUNTIME_DIR=/mnt/hdfs/yyding NNODES=8 GEN_TP=4 -CP=4 -ROLLOUT_RS=null TEST_FREQ=-1 project_name=${PROJECT_NAME:-'Uni-Agent-Qwen3-Coder-30B-megatron'} exp_name=${EXP_NAME:-"$(date +%Y%m%d%H)_exp"} -RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} -MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-Coder-30B-A3B-Instruct"} -CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} -AGENT_LOG_DIR=${AGENT_LOG_DIR:-"${RAY_DATA_HOME}/logs/${project_name}/${exp_name}"} -TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/uni_agent/swe_rebench_filtered_1150.parquet"} -TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/uni_agent/swe_bench_verified.parquet"} -RUNTIME_ENV=${RUNTIME_ENV:-"${RAY_DATA_HOME}/data/uni_agent/runtime_env.yaml"} +MODEL_PATH=${MODEL_PATH:-"${DATA_DIR}/models/Qwen3-Coder-30B-A3B-Instruct"} +TRAIN_FILE=${TRAIN_FILE:-"${DATA_DIR}/data/uni_agent/swe_rebench_filtered_1150.parquet"} +TEST_FILE=${TEST_FILE:-"${DATA_DIR}/data/uni_agent/swe_bench_verified.parquet"} + +RUNTIME_ENV=${RUNTIME_ENV:-"${RUNTIME_DIR}/data/uni_agent/runtime_env.yaml"} +CKPTS_DIR=${CKPTS_DIR:-"${RUNTIME_DIR}/ckpts/${project_name}/${exp_name}"} +AGENT_LOG_DIR=${AGENT_LOG_DIR:-"${RUNTIME_DIR}/logs/${project_name}/${exp_name}"} # Must be launched from the repository root so Ray packages both `verl/` and `uni_agent/`. # --- Agent-framework rollout (replaces the swe_agent agent-loop) -------------- # Run-wide task base (agent + sandbox + sampling), loaded from this YAML by @@ -40,8 +39,8 @@ kl_coef=${KL_COEF:-0.0} use_kl_loss=${USE_KL_LOSS:-False} kl_loss_coef=${KL_LOSS_COEF:-0.0} -clip_ratio_low=${CLIP_RATIO_LOW:-4e-4} -clip_ratio_high=${CLIP_RATIO_HIGH:-4e-4} +clip_ratio_low=${CLIP_RATIO_LOW:-0.2} +clip_ratio_high=${CLIP_RATIO_HIGH:-0.28} # Response length parameters max_prompt_length=${MAX_PROMPT_LENGTH:-$((1024 * 8))} @@ -51,7 +50,7 @@ overlong_buffer_len=${OVERLONG_BUFFER_LEN:-$((1024 * 4))} # unused overlong_penalty_factor=${OVERLONG_PENALTY_FACTOR:-1.0} loss_agg_mode=${LOSS_AGG_MODE:-"token-mean"} -loss_mode=${LOSS_MODE:-gspo} +loss_mode=${LOSS_MODE:-bypass_mode} # Algorithm temperature=${TEMPERATURE:-1.0} @@ -97,20 +96,23 @@ lr_decay_steps=${LR_DECAY_STEPS:-2000} test_freq=${TEST_FREQ:-10} # ============================================================================ -# Decoupled PPO (bypass_mode=False) + Rollout Correction (Rollout IS) +# Behavior-policy anchored PPO: ratio = pi_train / pi_rollout. +# The PPO ratio is both the importance-sampling ratio and the proximal ratio, +# so explicit rollout IS weights stay disabled to avoid double counting. # ============================================================================ -bypass_mode=${BYPASS_MODE:-False} # False => decoupled PPO (recompute old_log_prob as proximal anchor) -rollout_is=${ROLLOUT_IS:-token} # token | sequence | null (IS aggregation level) +bypass_mode=${BYPASS_MODE:-True} # True => old_log_prob = rollout_log_prob +bypass_loss_type=${BYPASS_LOSS_TYPE:-ppo_clip} # ppo_clip | reinforce +rollout_is=${ROLLOUT_IS:-null} # PPO clip already applies the IS ratio rollout_is_threshold=${ROLLOUT_IS_THRESHOLD:-2.0} # single float => TIS upper clamp; "lo_hi" string => IcePop rollout_is_batch_normalize=${ROLLOUT_IS_BATCH_NORMALIZE:-False} # normalize IS weights to mean=1.0 within a batch -rollout_rs=${ROLLOUT_RS:-seq_mean_k1} # seq_mean_k1 | seq_mean_k3 | token_k1 | null -rollout_rs_threshold=${ROLLOUT_RS_THRESHOLD:-"0.999_1.001"} # k1: "lo_hi" ratio band; k3: single upper bound +rollout_rs=${ROLLOUT_RS:-null} # no rejection sampling +rollout_rs_threshold=${ROLLOUT_RS_THRESHOLD:-null} # ============================================================================ -# 30B MoE Router Replay (R3) +# 30B MoE Router Replay # ============================================================================ -router_replay_mode=${ROUTER_REPLAY_MODE:-R3} # disabled | R2 | R3 -enable_rollout_routing_replay=${ENABLE_ROLLOUT_ROUTING_REPLAY:-True} # required for R3 (rollout-side replay) +router_replay_mode=${ROUTER_REPLAY_MODE:-disabled} # disabled | R2 | R3 +enable_rollout_routing_replay=${ENABLE_ROLLOUT_ROUTING_REPLAY:-False} # required only for R3 ray job submit --no-wait --runtime-env $RUNTIME_ENV \ -- python3 -m verl.trainer.main_ppo \ @@ -182,7 +184,15 @@ ray job submit --no-wait --runtime-env $RUNTIME_ENV \ algorithm.rollout_correction.rollout_is_batch_normalize=${rollout_is_batch_normalize} \ algorithm.rollout_correction.rollout_rs=${rollout_rs} \ algorithm.rollout_correction.rollout_rs_threshold="${rollout_rs_threshold}" \ - actor_rollout_ref.actor.router_replay.mode=${router_replay_mode} \ + algorithm.rollout_correction.loss_type=${bypass_loss_type} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.bypass_mode=${bypass_mode} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is=${rollout_is} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is_threshold=${rollout_is_threshold} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is_batch_normalize=${rollout_is_batch_normalize} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_rs=${rollout_rs} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_rs_threshold="${rollout_rs_threshold}" \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.loss_type=${bypass_loss_type} \ + actor_rollout_ref.actor.megatron.router_replay.mode=${router_replay_mode} \ actor_rollout_ref.rollout.enable_rollout_routing_replay=${enable_rollout_routing_replay} \ actor_rollout_ref.actor.entropy_coeff=0 \ actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ @@ -196,7 +206,7 @@ ray job submit --no-wait --runtime-env $RUNTIME_ENV \ ++actor_rollout_ref.rollout.custom.agent_framework.gateway_count=${GATEWAY_COUNT} \ ++actor_rollout_ref.rollout.custom.agent_framework.log_dir=${AGENT_LOG_DIR} \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_fqn=uni_agent.framework.task_runner.run_task \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.dispatch_mode=inline_async \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.dispatch_mode=ray_task \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.max_concurrent_sessions=${CONCURRENCY} \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.task_config_path=${TASK_CONFIG} \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.model_name=${SERVED_MODEL_NAME} \ diff --git a/examples/quickstart/training/task_config_claude_code.yaml b/examples/quickstart/training/task_config_claude_code.yaml new file mode 100644 index 00000000..7aa08b5f --- /dev/null +++ b/examples/quickstart/training/task_config_claude_code.yaml @@ -0,0 +1,25 @@ +- name: swe_bench + sandbox: + provider: vefaas + runtime_timeout: 7200 + agent: + name: claude_code + max_turns: 200 + run_timeout: 7200 + model: + temperature: 1.0 + top_p: 1.0 + max_total_tokens: 131072 + +- name: swe_rebench + sandbox: + provider: vefaas + runtime_timeout: 7200 + agent: + name: claude_code + max_turns: 200 + run_timeout: 7200 + model: + temperature: 1.0 + top_p: 1.0 + max_total_tokens: 131072 diff --git a/examples/quickstart/training/task_config_react.yaml b/examples/quickstart/training/task_config_react.yaml new file mode 100644 index 00000000..db2e72b7 --- /dev/null +++ b/examples/quickstart/training/task_config_react.yaml @@ -0,0 +1,45 @@ +- name: swe_bench + sandbox: + provider: vefaas + runtime_timeout: 7200 + agent: + name: react + max_steps: 200 + tools: + - name: stateful_shell + command_timeout: 120 + env_vars: + PAGER: "cat" + GIT_PAGER: "cat" + MANPAGER: "cat" + TQDM_DISABLE: "1" + PIP_PROGRESS_BAR: "off" + - name: str_replace_editor + - name: submit + model: + temperature: 1.0 + top_p: 1.0 + max_total_tokens: 131072 + +- name: swe_rebench + sandbox: + provider: vefaas + runtime_timeout: 7200 + agent: + name: react + max_steps: 200 + tools: + - name: stateful_shell + command_timeout: 120 + env_vars: + PAGER: "cat" + GIT_PAGER: "cat" + MANPAGER: "cat" + TQDM_DISABLE: "1" + PIP_PROGRESS_BAR: "off" + - name: str_replace_editor + - name: submit + model: + temperature: 1.0 + top_p: 1.0 + max_total_tokens: 131072 diff --git a/examples/quickstart/training/train_qwen3_moe.sh b/examples/quickstart/training/train_qwen3_moe.sh new file mode 100644 index 00000000..b338ad77 --- /dev/null +++ b/examples/quickstart/training/train_qwen3_moe.sh @@ -0,0 +1,261 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +DATA_DIR=/mnt/hdfs/yyding +RUNTIME_DIR=/mnt/hdfs/yyding +NNODES=8 +GEN_TP=4 +TEST_FREQ=-1 + +project_name=${PROJECT_NAME:-'Uni-Agent-Qwen3-Coder-30B-megatron'} +exp_name=${EXP_NAME:-"$(date +%Y%m%d%H)_exp"} + +MODEL_PATH=${MODEL_PATH:-"${DATA_DIR}/models/Qwen3-Coder-30B-A3B-Instruct"} +TRAIN_FILE=${TRAIN_FILE:-"${DATA_DIR}/data/uni_agent/swe_rebench_filtered_1150.parquet"} +TEST_FILE=${TEST_FILE:-"${DATA_DIR}/data/uni_agent/swe_bench_verified.parquet"} + +RUNTIME_ENV=${RUNTIME_ENV:-"${RUNTIME_DIR}/data/uni_agent/runtime_env.yaml"} +CKPTS_DIR=${CKPTS_DIR:-"${RUNTIME_DIR}/ckpts/${project_name}/${exp_name}"} +AGENT_LOG_DIR=${AGENT_LOG_DIR:-"${RUNTIME_DIR}/logs/${project_name}/${exp_name}"} +# Must be launched from the repository root so Ray packages both `verl/` and `uni_agent/`. +# --- Agent-framework rollout (replaces the swe_agent agent-loop) -------------- +# Run-wide task base (agent + sandbox + sampling), loaded from this YAML by +# uni_agent.framework.task_runner.run_task and deep-merged onto each row's task. +# Same file-path idea as the old agent_loop_config_path; new (task-config) schema. +TASK_CONFIG=${TASK_CONFIG:-"examples/quickstart/training/task_config_react.yaml"} +TOOL_PARSER=${TOOL_PARSER:-"qwen3_coder"} # gateway tool-call parser; MUST match the model chat template +GATEWAY_COUNT=${GATEWAY_COUNT:-8} # gateway actors fronting the engine +CONCURRENCY=${CONCURRENCY:-512} # max in-flight rollout sessions (runner cap) +SERVED_MODEL_NAME=${SERVED_MODEL_NAME:-"$(basename "${MODEL_PATH}")"} + +rollout_mode=${ROLLOUT_MODE:-"async"} +rollout_name=${ROLLOUT_NAME:-"vllm"} # sglang or vllm + +# Algorithm parameters +adv_estimator=${ADV_ESTIMATOR:-grpo} + +use_kl_in_reward=${USE_KL_IN_REWARD:-False} +kl_coef=${KL_COEF:-0.0} +use_kl_loss=${USE_KL_LOSS:-False} +kl_loss_coef=${KL_LOSS_COEF:-0.0} + +clip_ratio_low=${CLIP_RATIO_LOW:-0.2} +clip_ratio_high=${CLIP_RATIO_HIGH:-0.28} + +# Response length parameters +max_prompt_length=${MAX_PROMPT_LENGTH:-$((1024 * 8))} +max_response_length=${MAX_RESPONSE_LENGTH:-$((1024 * 128))} +enable_overlong_buffer=${ENABLE_OVERLONG_BUFFER:-False} +overlong_buffer_len=${OVERLONG_BUFFER_LEN:-$((1024 * 4))} # unused +overlong_penalty_factor=${OVERLONG_PENALTY_FACTOR:-1.0} + +loss_agg_mode=${LOSS_AGG_MODE:-"token-mean"} +loss_mode=${LOSS_MODE:-bypass_mode} + +# Algorithm +temperature=${TEMPERATURE:-1.0} +top_p=${TOP_P:-1.0} +top_k=${TOP_K:--1} +val_temperature=${VAL_TEMPERATURE:-1.0} +val_top_p=${VAL_TOP_P:-0.95} +val_top_k=${VAL_TOP_K:--1} + +# Performance Related Parameter +use_dynamic_bsz=${USE_DYNAMIC_BSZ:-True} +offload=${OFFLOAD:-True} +gen_tp=${GEN_TP:-4} +train_tp=${TP:-4} +train_pp=${PP:-2} +train_cp=${CP:-2} +train_ep=${EP:-8} +train_etp=${ETP:-1} +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / train_cp)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / train_cp)) + +optimizer_offload_fraction=${OFFLOAD_FRACTION:-1.0} + +# install mbridge +# pip3 install git+https://github.com/ISEEKYAN/mbridge +USE_MBRIDGE=${USE_MBRIDGE:-True} +USE_DIST_CKPT=${USE_DIST_CKPT:-False} + +# V1 colocate_async topology. colocate_async colocates actor + rollout on the same +# GPUs (rollout replicas sleep during the train step), so NNODES is the TOTAL node +# count (replaces the old fully-async NNODES_ROLLOUT + NNODES_TRAIN split). +NNODES=${NNODES:-8} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} + +# parameter_sync_step defaults to 1 for colocate_async, so train_batch_size +# (prompts/step) only needs to be > 0 (the old async mode used train_batch_size=0). +# num_warmup_batches pre-fills the rollout pipeline before the first train step. +train_prompt_bsz=${TRAIN_PROMPT_BSZ:-64} +n_resp_per_prompt=${N_RESP_PER_PROMPT:-8} +train_prompt_mini_bsz=${PPO_MINI_BATCH_SIZE:-16} +num_warmup_batches=${NUM_WARMUP_BATCHES:-1} +lr_decay_steps=${LR_DECAY_STEPS:-2000} +test_freq=${TEST_FREQ:-10} + +# ============================================================================ +# Behavior-policy anchored PPO: ratio = pi_train / pi_rollout. +# The PPO ratio is both the importance-sampling ratio and the proximal ratio, +# so explicit rollout IS weights stay disabled to avoid double counting. +# ============================================================================ +bypass_mode=${BYPASS_MODE:-True} # True => old_log_prob = rollout_log_prob +bypass_loss_type=${BYPASS_LOSS_TYPE:-ppo_clip} # ppo_clip | reinforce +rollout_is=${ROLLOUT_IS:-null} # PPO clip already applies the IS ratio +rollout_is_threshold=${ROLLOUT_IS_THRESHOLD:-2.0} # single float => TIS upper clamp; "lo_hi" string => IcePop +rollout_is_batch_normalize=${ROLLOUT_IS_BATCH_NORMALIZE:-False} # normalize IS weights to mean=1.0 within a batch +rollout_rs=${ROLLOUT_RS:-null} # no rejection sampling +rollout_rs_threshold=${ROLLOUT_RS_THRESHOLD:-null} + +# ============================================================================ +# 30B MoE Router Replay +# ============================================================================ +router_replay_mode=${ROUTER_REPLAY_MODE:-disabled} # disabled | R2 | R3 +enable_rollout_routing_replay=${ENABLE_ROLLOUT_ROUTING_REPLAY:-False} # required only for R3 + +ray job submit --no-wait --runtime-env $RUNTIME_ENV \ + -- python3 -m verl.trainer.main_ppo \ + --config-name=ppo_megatron_trainer \ + trainer.use_v1=True \ + trainer.v1.trainer_mode=colocate_async \ + trainer.v1.colocate_async.num_warmup_batches=${num_warmup_batches} \ + transfer_queue.enable=True \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + data.return_raw_chat=True \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_decay_style='constant' \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.optim.lr_decay_steps=${lr_decay_steps} \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=${optimizer_offload_fraction} \ + +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \ + actor_rollout_ref.actor.megatron.use_mbridge=$USE_MBRIDGE \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=$USE_DIST_CKPT \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.context_parallel_size=${train_cp} \ + actor_rollout_ref.actor.megatron.expert_model_parallel_size=${train_ep} \ + actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=${train_etp} \ + +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.masked_softmax_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_activation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_dropout_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.deallocate_pipeline_outputs=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.persist_layer_norm=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_token_dispatcher_type="alltoall" \ + +actor_rollout_ref.actor.megatron.override_transformer_config.moe_router_dtype=fp32 \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_method=uniform \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity=full \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_num_layers=1 \ + algorithm.rollout_correction.bypass_mode=${bypass_mode} \ + algorithm.rollout_correction.rollout_is=${rollout_is} \ + algorithm.rollout_correction.rollout_is_threshold=${rollout_is_threshold} \ + algorithm.rollout_correction.rollout_is_batch_normalize=${rollout_is_batch_normalize} \ + algorithm.rollout_correction.rollout_rs=${rollout_rs} \ + algorithm.rollout_correction.rollout_rs_threshold="${rollout_rs_threshold}" \ + algorithm.rollout_correction.loss_type=${bypass_loss_type} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.bypass_mode=${bypass_mode} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is=${rollout_is} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is_threshold=${rollout_is_threshold} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is_batch_normalize=${rollout_is_batch_normalize} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_rs=${rollout_rs} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_rs_threshold="${rollout_rs_threshold}" \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.loss_type=${bypass_loss_type} \ + actor_rollout_ref.actor.megatron.router_replay.mode=${router_replay_mode} \ + actor_rollout_ref.rollout.enable_rollout_routing_replay=${enable_rollout_routing_replay} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + +actor_rollout_ref.actor.checkpoint.save_contents=['model','hf_model'] \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.multi_turn.enable=True \ + actor_rollout_ref.rollout.multi_turn.max_parallel_calls=1 \ + ++actor_rollout_ref.rollout.multi_turn.format=${TOOL_PARSER} \ + actor_rollout_ref.rollout.agent.num_workers=8 \ + ++actor_rollout_ref.rollout.agent.agent_loop_manager_class=uni_agent.framework.entry.AgentFrameworkRolloutAdapter \ + ++actor_rollout_ref.rollout.custom.agent_framework.gateway_count=${GATEWAY_COUNT} \ + ++actor_rollout_ref.rollout.custom.agent_framework.log_dir=${AGENT_LOG_DIR} \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_fqn=uni_agent.framework.task_runner.run_task \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.dispatch_mode=ray_task \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.max_concurrent_sessions=${CONCURRENCY} \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.task_config_path=${TASK_CONFIG} \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.model_name=${SERVED_MODEL_NAME} \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.report_reward=True \ + ++actor_rollout_ref.rollout.custom.agent_framework.use_reward_loop_worker=False \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.prompt_length=${max_prompt_length} \ + actor_rollout_ref.rollout.response_length=${max_response_length} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.max_model_len=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${val_temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${val_top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=${rollout_name} \ + actor_rollout_ref.rollout.mode=${rollout_mode} \ + actor_rollout_ref.rollout.calculate_log_probs=True \ + actor_rollout_ref.nccl_timeout=9600 \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.context_parallel_size=${train_cp} \ + actor_rollout_ref.ref.megatron.expert_model_parallel_size=${train_ep} \ + actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=${train_etp} \ + reward.reward_manager.name=dapo \ + +reward.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','wandb'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.val_before_train=False \ + trainer.save_freq=10 \ + trainer.total_epochs=10 \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.nnodes="${NNODES}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.test_freq="${test_freq}" From 3ac376df892ee633d33bd0d56776447fce47f6c4 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Thu, 23 Jul 2026 01:32:56 +0800 Subject: [PATCH 02/11] update --- docs/source/quickstart/rl-training.md | 78 +++++- examples/inference/runtime_env.yaml | 4 +- .../quickstart/training/train_qwen3_moe.sh | 8 +- .../training/train_qwen3p5_dense.sh | 237 ++++++++++++++++++ .../agents/test_claude_code_agent.py | 12 + uni_agent/agents/claude_code/agent.py | 44 ++-- 6 files changed, 357 insertions(+), 26 deletions(-) create mode 100644 examples/quickstart/training/train_qwen3p5_dense.sh diff --git a/docs/source/quickstart/rl-training.md b/docs/source/quickstart/rl-training.md index 1062ec2d..61790684 100644 --- a/docs/source/quickstart/rl-training.md +++ b/docs/source/quickstart/rl-training.md @@ -3,7 +3,7 @@ This guide demonstrates Agentic RL training for both white-box and black-box agents: 1. Train `Qwen3-Coder-30B-A3B-Instruct` with the white-box `ReAct Agent`. -2. Train `Qwen3.5-9B` with the black-box `Claude Code` Agent. +2. Train `Qwen3.5-4B` with the black-box `Claude Code` Agent. ## Prerequisites @@ -182,14 +182,90 @@ Training runs as a Ray job. Use a Runtime Environment to distribute the reposito ### Launch Training +This recipe trains `Qwen3-Coder-30B-A3B-Instruct` with the ReAct Task Config. Set the shared data and runtime roots, then launch it from the repository root: + +```bash +DATA_DIR=/path/to/data \ +RUNTIME_DIR=/path/to/runtime \ +NNODES=8 \ +ADV_ESTIMATOR=rloo \ +TASK_CONFIG=examples/quickstart/training/task_config_react.yaml \ +EXP_NAME=react_qwen3_coder_30b \ +bash examples/quickstart/training/train_qwen3_moe.sh +``` + +The default layout is: + +```text +/ +├── models/Qwen3-Coder-30B-A3B-Instruct/ +└── data/uni_agent/ + ├── swe_rebench_filtered_1150.parquet + └── swe_bench_verified.parquet + +/ +├── data/uni_agent/runtime_env.yaml +├── ckpts/ +└── logs/ +``` + +Override `MODEL_PATH`, `TRAIN_FILE`, `TEST_FILE`, `RUNTIME_ENV`, or `TASK_CONFIG` when your layout differs. + ### Monitor the Run +Checkpoints and per-session Agent logs are written under: + +```text +/ckpts/Uni-Agent-Qwen3-Coder-30B-megatron// +/logs/Uni-Agent-Qwen3-Coder-30B-megatron// +``` + ### Results +_To be added._ + ## Case 2: Claude Code RL ### Launch Training +This recipe trains `Qwen3.5-4B` with the Claude Code Task Config: + +```bash +DATA_DIR=/path/to/data \ +RUNTIME_DIR=/path/to/runtime \ +NNODES=8 \ +ADV_ESTIMATOR=rloo \ +TASK_CONFIG=examples/quickstart/training/task_config_claude_code.yaml \ +EXP_NAME=claude_code_qwen3_5_4b \ +bash examples/quickstart/training/train_qwen3p5_dense.sh +``` + +The script expects: + +```text +/ +├── models/Qwen3.5-4B/ +└── data/uni_agent/ + ├── swe_rebench_filtered_1150.parquet + └── swe_bench_verified.parquet + +/ +├── data/uni_agent/runtime_env.yaml +├── ckpts/ +└── logs/ +``` + +The Claude Code sandbox must be able to reach the session-scoped Gateway running on the GPU cluster. + ### Monitor the Run +Outputs are written under: + +```text +/ckpts/Uni-Agent-Qwen3.5-4B-megatron// +/logs/Uni-Agent-Qwen3.5-4B-megatron// +``` + ### Results + +_To be added._ diff --git a/examples/inference/runtime_env.yaml b/examples/inference/runtime_env.yaml index 99a16b78..2b3e6f3f 100644 --- a/examples/inference/runtime_env.yaml +++ b/examples/inference/runtime_env.yaml @@ -15,5 +15,5 @@ env_vars: VOLCE_ACCESS_KEY: "xxxxxx" VOLCE_SECRET_KEY: "xxxxxx" # if you use modal, set the following variables - MODAL_TOKEN_ID: "ak-MWuXc0JB73Ll3y75lnTl1K" - MODAL_TOKEN_SECRET: "as-b3iq9Xf3igKAb3DfPQVNwG" + MODAL_TOKEN_ID: "" + MODAL_TOKEN_SECRET: "" diff --git a/examples/quickstart/training/train_qwen3_moe.sh b/examples/quickstart/training/train_qwen3_moe.sh index b338ad77..849e5f1c 100644 --- a/examples/quickstart/training/train_qwen3_moe.sh +++ b/examples/quickstart/training/train_qwen3_moe.sh @@ -1,13 +1,7 @@ #!/usr/bin/env bash set -xeuo pipefail -DATA_DIR=/mnt/hdfs/yyding -RUNTIME_DIR=/mnt/hdfs/yyding -NNODES=8 -GEN_TP=4 -TEST_FREQ=-1 - -project_name=${PROJECT_NAME:-'Uni-Agent-Qwen3-Coder-30B-megatron'} +project_name=${PROJECT_NAME:-"Uni-Agent-Qwen3-Coder-30B-megatron"} exp_name=${EXP_NAME:-"$(date +%Y%m%d%H)_exp"} MODEL_PATH=${MODEL_PATH:-"${DATA_DIR}/models/Qwen3-Coder-30B-A3B-Instruct"} diff --git a/examples/quickstart/training/train_qwen3p5_dense.sh b/examples/quickstart/training/train_qwen3p5_dense.sh new file mode 100644 index 00000000..e4c09d1a --- /dev/null +++ b/examples/quickstart/training/train_qwen3p5_dense.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +project_name=${PROJECT_NAME:-"Uni-Agent-Qwen3.5-4B-megatron"} +exp_name=${EXP_NAME:-"$(date +%Y%m%d%H)_exp"} + +MODEL_PATH=${MODEL_PATH:-"${DATA_DIR}/models/Qwen3.5-4B"} +TRAIN_FILE=${TRAIN_FILE:-"${DATA_DIR}/data/uni_agent/swe_rebench_filtered_1150.parquet"} +TEST_FILE=${TEST_FILE:-"${DATA_DIR}/data/uni_agent/swe_bench_verified.parquet"} + +RUNTIME_ENV=${RUNTIME_ENV:-"${RUNTIME_DIR}/data/uni_agent/runtime_env.yaml"} +CKPTS_DIR=${CKPTS_DIR:-"${RUNTIME_DIR}/ckpts/${project_name}/${exp_name}"} +AGENT_LOG_DIR=${AGENT_LOG_DIR:-"${RUNTIME_DIR}/logs/${project_name}/${exp_name}"} +# Must be launched from the repository root so Ray packages both `verl/` and `uni_agent/`. +# --- Agent-framework rollout (replaces the swe_agent agent-loop) -------------- +# Run-wide task base (agent + sandbox + sampling), loaded from this YAML by +# uni_agent.framework.task_runner.run_task and deep-merged onto each row's task. +# Same file-path idea as the old agent_loop_config_path; new (task-config) schema. +TASK_CONFIG=${TASK_CONFIG:-"examples/quickstart/training/task_config_claude_code.yaml"} +TOOL_PARSER=${TOOL_PARSER:-"qwen3_coder"} # gateway tool-call parser; MUST match the model chat template +GATEWAY_COUNT=${GATEWAY_COUNT:-8} # gateway actors fronting the engine +CONCURRENCY=${CONCURRENCY:-256} # max in-flight rollout sessions (runner cap) +SERVED_MODEL_NAME=${SERVED_MODEL_NAME:-"$(basename "${MODEL_PATH}")"} + +rollout_mode=${ROLLOUT_MODE:-"async"} +rollout_name=${ROLLOUT_NAME:-"vllm"} # sglang or vllm + +# Algorithm parameters +adv_estimator=${ADV_ESTIMATOR:-grpo} + +use_kl_in_reward=${USE_KL_IN_REWARD:-False} +kl_coef=${KL_COEF:-0.0} +use_kl_loss=${USE_KL_LOSS:-False} +kl_loss_coef=${KL_LOSS_COEF:-0.0} + +clip_ratio_low=${CLIP_RATIO_LOW:-0.2} +clip_ratio_high=${CLIP_RATIO_HIGH:-0.28} + +# Response length parameters +max_prompt_length=${MAX_PROMPT_LENGTH:-$((1024 * 8))} +max_response_length=${MAX_RESPONSE_LENGTH:-$((1024 * 128))} +enable_overlong_buffer=${ENABLE_OVERLONG_BUFFER:-False} +overlong_buffer_len=${OVERLONG_BUFFER_LEN:-$((1024 * 4))} # unused +overlong_penalty_factor=${OVERLONG_PENALTY_FACTOR:-1.0} + +loss_agg_mode=${LOSS_AGG_MODE:-"token-mean"} +loss_mode=${LOSS_MODE:-bypass_mode} + +# Algorithm +temperature=${TEMPERATURE:-1.0} +top_p=${TOP_P:-1.0} +top_k=${TOP_K:--1} +val_temperature=${VAL_TEMPERATURE:-1.0} +val_top_p=${VAL_TOP_P:-0.95} +val_top_k=${VAL_TOP_K:--1} + +# Performance Related Parameter +use_dynamic_bsz=${USE_DYNAMIC_BSZ:-True} +offload=${OFFLOAD:-False} +gen_tp=${GEN_TP:-2} +train_tp=${TP:-4} +train_pp=${PP:-1} +train_cp=${CP:-2} +actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / train_cp)) +infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / train_cp)) + +optimizer_offload_fraction=${OFFLOAD_FRACTION:-1.0} + +# install mbridge +# pip3 install git+https://github.com/ISEEKYAN/mbridge +USE_MBRIDGE=${USE_MBRIDGE:-True} +USE_DIST_CKPT=${USE_DIST_CKPT:-False} + +# V1 colocate_async topology. colocate_async colocates actor + rollout on the same +# GPUs (rollout replicas sleep during the train step), so NNODES is the TOTAL node +# count (replaces the old fully-async NNODES_ROLLOUT + NNODES_TRAIN split). +NNODES=${NNODES:-8} +NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} + +# parameter_sync_step defaults to 1 for colocate_async, so train_batch_size +# (prompts/step) only needs to be > 0 (the old async mode used train_batch_size=0). +# num_warmup_batches pre-fills the rollout pipeline before the first train step. +train_prompt_bsz=${TRAIN_PROMPT_BSZ:-32} +n_resp_per_prompt=${N_RESP_PER_PROMPT:-8} +train_prompt_mini_bsz=${PPO_MINI_BATCH_SIZE:-16} +num_warmup_batches=${NUM_WARMUP_BATCHES:-1} +lr_decay_steps=${LR_DECAY_STEPS:-2000} +test_freq=${TEST_FREQ:-10} + +# ============================================================================ +# Behavior-policy anchored PPO: ratio = pi_train / pi_rollout. +# The PPO ratio is both the importance-sampling ratio and the proximal ratio, +# so explicit rollout IS weights stay disabled to avoid double counting. +# ============================================================================ +bypass_mode=${BYPASS_MODE:-True} # True => old_log_prob = rollout_log_prob +bypass_loss_type=${BYPASS_LOSS_TYPE:-ppo_clip} # ppo_clip | reinforce +rollout_is=${ROLLOUT_IS:-null} # PPO clip already applies the IS ratio +rollout_is_threshold=${ROLLOUT_IS_THRESHOLD:-2.0} # single float => TIS upper clamp; "lo_hi" string => IcePop +rollout_is_batch_normalize=${ROLLOUT_IS_BATCH_NORMALIZE:-False} # normalize IS weights to mean=1.0 within a batch +rollout_rs=${ROLLOUT_RS:-null} # no rejection sampling +rollout_rs_threshold=${ROLLOUT_RS_THRESHOLD:-null} + +ray job submit --no-wait --runtime-env $RUNTIME_ENV \ + -- python3 -m verl.trainer.main_ppo \ + --config-name=ppo_megatron_trainer \ + trainer.use_v1=True \ + trainer.v1.trainer_mode=colocate_async \ + trainer.v1.colocate_async.num_warmup_batches=${num_warmup_batches} \ + transfer_queue.enable=True \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.train_batch_size=${train_prompt_bsz} \ + data.return_raw_chat=True \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.model.use_fused_kernels=False \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_decay_style='constant' \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.optim.lr_decay_steps=${lr_decay_steps} \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=${optimizer_offload_fraction} \ + +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \ + +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \ + actor_rollout_ref.actor.megatron.use_mbridge=$USE_MBRIDGE \ + actor_rollout_ref.actor.megatron.use_dist_checkpointing=$USE_DIST_CKPT \ + actor_rollout_ref.actor.megatron.param_offload=${offload} \ + actor_rollout_ref.actor.megatron.grad_offload=${offload} \ + actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ + actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.actor.megatron.context_parallel_size=${train_cp} \ + +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=False \ + +actor_rollout_ref.actor.megatron.override_transformer_config.masked_softmax_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_activation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.bias_dropout_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.deallocate_pipeline_outputs=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.persist_layer_norm=True \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_method=uniform \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity=full \ + +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_num_layers=1 \ + algorithm.rollout_correction.bypass_mode=${bypass_mode} \ + algorithm.rollout_correction.rollout_is=${rollout_is} \ + algorithm.rollout_correction.rollout_is_threshold=${rollout_is_threshold} \ + algorithm.rollout_correction.rollout_is_batch_normalize=${rollout_is_batch_normalize} \ + algorithm.rollout_correction.rollout_rs=${rollout_rs} \ + algorithm.rollout_correction.rollout_rs_threshold="${rollout_rs_threshold}" \ + algorithm.rollout_correction.loss_type=${bypass_loss_type} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.bypass_mode=${bypass_mode} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is=${rollout_is} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is_threshold=${rollout_is_threshold} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is_batch_normalize=${rollout_is_batch_normalize} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_rs=${rollout_rs} \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_rs_threshold="${rollout_rs_threshold}" \ + ++actor_rollout_ref.actor.policy_loss.rollout_correction.loss_type=${bypass_loss_type} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + +actor_rollout_ref.actor.checkpoint.save_contents=['model','hf_model'] \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.rollout.multi_turn.enable=True \ + actor_rollout_ref.rollout.multi_turn.max_parallel_calls=1 \ + ++actor_rollout_ref.rollout.multi_turn.format=${TOOL_PARSER} \ + actor_rollout_ref.rollout.agent.num_workers=8 \ + ++actor_rollout_ref.rollout.agent.agent_loop_manager_class=uni_agent.framework.entry.AgentFrameworkRolloutAdapter \ + ++actor_rollout_ref.rollout.custom.agent_framework.gateway_count=${GATEWAY_COUNT} \ + ++actor_rollout_ref.rollout.custom.agent_framework.log_dir=${AGENT_LOG_DIR} \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_fqn=uni_agent.framework.task_runner.run_task \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.dispatch_mode=ray_task \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.max_concurrent_sessions=${CONCURRENCY} \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.task_config_path=${TASK_CONFIG} \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.model_name=${SERVED_MODEL_NAME} \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.report_reward=True \ + ++actor_rollout_ref.rollout.custom.agent_framework.use_reward_loop_worker=False \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ + actor_rollout_ref.rollout.prompt_length=${max_prompt_length} \ + actor_rollout_ref.rollout.response_length=${max_response_length} \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.max_model_len=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.temperature=${val_temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${val_top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.name=${rollout_name} \ + actor_rollout_ref.rollout.mode=${rollout_mode} \ + actor_rollout_ref.rollout.calculate_log_probs=True \ + actor_rollout_ref.nccl_timeout=9600 \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ + actor_rollout_ref.ref.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \ + actor_rollout_ref.ref.megatron.param_offload=${offload} \ + actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ + actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ + actor_rollout_ref.ref.megatron.context_parallel_size=${train_cp} \ + reward.reward_manager.name=dapo \ + +reward.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ + +reward.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ + +reward.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ + +reward.reward_kwargs.overlong_buffer_cfg.log=False \ + +reward.reward_kwargs.max_resp_len=${max_response_length} \ + trainer.logger=['console','wandb'] \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.val_before_train=False \ + trainer.save_freq=10 \ + trainer.total_epochs=10 \ + trainer.resume_mode=auto \ + trainer.log_val_generations=10 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.nnodes="${NNODES}" \ + trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ + trainer.test_freq="${test_freq}" diff --git a/tests/uni_agent/agents/test_claude_code_agent.py b/tests/uni_agent/agents/test_claude_code_agent.py index fb711bf7..73acc902 100644 --- a/tests/uni_agent/agents/test_claude_code_agent.py +++ b/tests/uni_agent/agents/test_claude_code_agent.py @@ -130,9 +130,21 @@ def test_run_uses_sandbox_default_workdir(): assert len(sandbox.exec_calls) == 1 assert sandbox.exec_calls[0]["workdir"] is None + argv = sandbox.exec_calls[0]["argv"] + assert argv[:4] == ["claude", "--bare", "--no-session-persistence", "-p"] + assert argv[argv.index("--model") + 1] == "policy" + assert argv[argv.index("--permission-mode") + 1] == "bypassPermissions" + assert "--disable-slash-commands" in argv + assert "--dangerously-skip-permissions" not in argv + disallowed_tools = argv[argv.index("--disallowedTools") + 1].split(",") + assert set(disallowed_tools) == {"Agent", "Task", "WebFetch", "WebSearch", "AskUserQuestion"} assert sandbox.exec_calls[0]["env"]["ANTHROPIC_BASE_URL"] == "https://ark.example/api/compatible" assert sandbox.exec_calls[0]["env"]["ANTHROPIC_API_KEY"] == "" assert sandbox.exec_calls[0]["env"]["ANTHROPIC_AUTH_TOKEN"] == "ark-test-api-key" + assert sandbox.exec_calls[0]["env"]["CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS"] == "1" + assert sandbox.exec_calls[0]["env"]["CLAUDE_CODE_ATTRIBUTION_HEADER"] == "0" + assert sandbox.exec_calls[0]["env"]["CLAUDE_CODE_FORK_SUBAGENT"] == "0" + assert sandbox.exec_calls[0]["env"]["CLAUDE_CODE_SKIP_PROMPT_HISTORY"] == "1" def test_claude_env_uses_placeholders_for_session_gateway(): diff --git a/uni_agent/agents/claude_code/agent.py b/uni_agent/agents/claude_code/agent.py index f49a7248..9afb3aaa 100644 --- a/uni_agent/agents/claude_code/agent.py +++ b/uni_agent/agents/claude_code/agent.py @@ -26,14 +26,12 @@ _CC_QUIET_ENV = { "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS": "1", + "CLAUDE_CODE_ATTRIBUTION_HEADER": "0", "CLAUDE_CODE_DISABLE_BACKGROUND_TASKS": "1", - "CLAUDE_CODE_IDE_SKIP_AUTO_INSTALL": "1", + "CLAUDE_CODE_FORK_SUBAGENT": "0", + "CLAUDE_CODE_SKIP_PROMPT_HISTORY": "1", "CLAUDE_CODE_DISABLE_TERMINAL_TITLE": "1", - "DISABLE_AUTOUPDATER": "1", - "DISABLE_TELEMETRY": "1", - "DISABLE_ERROR_REPORTING": "1", - "DISABLE_BUG_COMMAND": "1", - "DISABLE_NON_ESSENTIAL_MODEL_CALLS": "1", } _CLAUDE_NPM_INSTALL_COMMAND = "npm install -g @anthropic-ai/claude-code --no-audit --no-fund" @@ -78,14 +76,16 @@ class ClaudeCodeConfig(AgentConfig): name: str = "claude_code" max_turns: int | None = Field(default=80, description="--max-turns budget; None to omit.") disallowed_tools: list[str] = Field( - default_factory=lambda: ["WebFetch", "WebSearch", "AskUserQuestion"], + default_factory=lambda: ["Agent", "Task", "WebFetch", "WebSearch", "AskUserQuestion"], description=( - "--disallowedTools deny-list. Under --dangerously-skip-permissions an *allow*-list is a " - "no-op (bypass approves every tool), so we DENY the tools that can't work in a headless, " - "offline sandbox: web tools (no egress) and AskUserQuestion (would hang on input). A bare " - "tool name drops it from Claude's context entirely, and deny wins even under bypass." + "--disallowedTools deny-list. Subagent, web, and interactive-user tools are disabled " + "to keep each headless rollout self-contained and deterministic." ), ) + permission_mode: str = Field( + default="bypassPermissions", + description="Claude Code --permission-mode used for unattended Sandbox execution.", + ) verbose: bool = Field(default=False, description="Pass --verbose (streams per-turn detail; noisy at scale).") run_timeout: float = Field(default=1800.0, description="Wallclock cap (s) on the claude process.") extra_args: list[str] = Field(default_factory=list, description="Extra flags appended to the claude argv.") @@ -159,7 +159,21 @@ def _split_messages(self, messages: list[dict[str, Any]]) -> tuple[str | None, s def _claude_argv(self, problem: str, system_prompt: str | None) -> list[str]: cfg: ClaudeCodeConfig = self.config # type: ignore[assignment] - argv = ["claude", "-p", problem] + model = cfg.model.model_name + if not model: + raise ValueError("claude_code: set config.model.model_name (the model claude sends)") + argv = [ + "claude", + "--bare", + "--no-session-persistence", + "-p", + problem, + "--model", + model, + "--permission-mode", + cfg.permission_mode, + "--disable-slash-commands", + ] if cfg.disallowed_tools: argv += ["--disallowedTools", ",".join(cfg.disallowed_tools)] if cfg.max_turns is not None: @@ -167,8 +181,6 @@ def _claude_argv(self, problem: str, system_prompt: str | None) -> list[str]: if system_prompt: # Append (don't replace) so Claude Code keeps its built-in tool/safety prompt. argv += ["--append-system-prompt", system_prompt] - # Headless runs must not block on permission prompts. - argv += ["--dangerously-skip-permissions"] if cfg.verbose: argv += ["--verbose"] return argv + list(cfg.extra_args) @@ -182,8 +194,8 @@ def _claude_env(self, endpoint: str) -> dict[str, str]: has_external_api_key = bool(configured_api_key and configured_api_key != "EMPTY") return { "ANTHROPIC_BASE_URL": endpoint, - # We always run inside a sandbox: lets `--dangerously-skip-permissions` run as - # root (else the CLI refuses) and skips its 529-overload guard path. + # We always run inside a sandbox: allows unattended permission bypass + # while running as root and skips Claude Code's overload guard path. "IS_SANDBOX": "1", # External endpoints receive ModelConfig's Bearer key. The session Gateway # ignores auth, but Claude Code still requires non-empty placeholder values. From 9a29c29b9444b761ea1cf273c597e91b651b06bd Mon Sep 17 00:00:00 2001 From: yuyangding Date: Thu, 23 Jul 2026 14:19:05 +0800 Subject: [PATCH 03/11] update --- docs/source/quickstart/rl-training.md | 15 +++++++++++++-- examples/quickstart/training/train_qwen3_moe.sh | 13 +++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/source/quickstart/rl-training.md b/docs/source/quickstart/rl-training.md index 61790684..33c3a69a 100644 --- a/docs/source/quickstart/rl-training.md +++ b/docs/source/quickstart/rl-training.md @@ -188,12 +188,23 @@ This recipe trains `Qwen3-Coder-30B-A3B-Instruct` with the ReAct Task Config. Se DATA_DIR=/path/to/data \ RUNTIME_DIR=/path/to/runtime \ NNODES=8 \ -ADV_ESTIMATOR=rloo \ +TP=1 PP=1 CP=4 EP=8 ETP=1 \ TASK_CONFIG=examples/quickstart/training/task_config_react.yaml \ -EXP_NAME=react_qwen3_coder_30b \ +EXP_NAME=react_qwen3_coder_30b_dppo_tv \ +ADV_ESTIMATOR=rloo \ +LOSS_MODE=dppo_tv \ +CLIP_RATIO_LOW=0.15 \ +CLIP_RATIO_HIGH=0.15 \ +CLIP_RATIO_C=10000 \ +LOSS_AGG_MODE=seq-mean-token-sum-norm \ +BYPASS_MODE=False \ +ROLLOUT_IS=null \ +ROLLOUT_RS=null \ bash examples/quickstart/training/train_qwen3_moe.sh ``` +This command uses TP1, PP1, CP4, and EP8 to split the 128K Agent context while avoiding tensor- and pipeline-parallel communication. It uses the DPPO-TV settings from the verl reference recipe. To run DPPO-KL instead, set `LOSS_MODE=dppo_kl` and use `CLIP_RATIO_LOW=0.05` and `CLIP_RATIO_HIGH=0.05`. + The default layout is: ```text diff --git a/examples/quickstart/training/train_qwen3_moe.sh b/examples/quickstart/training/train_qwen3_moe.sh index 849e5f1c..5079f557 100644 --- a/examples/quickstart/training/train_qwen3_moe.sh +++ b/examples/quickstart/training/train_qwen3_moe.sh @@ -35,6 +35,7 @@ kl_loss_coef=${KL_LOSS_COEF:-0.0} clip_ratio_low=${CLIP_RATIO_LOW:-0.2} clip_ratio_high=${CLIP_RATIO_HIGH:-0.28} +clip_ratio_c=${CLIP_RATIO_C:-10.0} # Response length parameters max_prompt_length=${MAX_PROMPT_LENGTH:-$((1024 * 8))} @@ -44,7 +45,7 @@ overlong_buffer_len=${OVERLONG_BUFFER_LEN:-$((1024 * 4))} # unused overlong_penalty_factor=${OVERLONG_PENALTY_FACTOR:-1.0} loss_agg_mode=${LOSS_AGG_MODE:-"token-mean"} -loss_mode=${LOSS_MODE:-bypass_mode} +loss_mode=${LOSS_MODE:-vanilla} # Algorithm temperature=${TEMPERATURE:-1.0} @@ -90,11 +91,11 @@ lr_decay_steps=${LR_DECAY_STEPS:-2000} test_freq=${TEST_FREQ:-10} # ============================================================================ -# Behavior-policy anchored PPO: ratio = pi_train / pi_rollout. -# The PPO ratio is both the importance-sampling ratio and the proximal ratio, -# so explicit rollout IS weights stay disabled to avoid double counting. +# Rollout correction is disabled by default for the standard GRPO + PPO +# baseline. Override these variables to enable behavior-anchor or decoupled +# rollout-correction experiments. # ============================================================================ -bypass_mode=${BYPASS_MODE:-True} # True => old_log_prob = rollout_log_prob +bypass_mode=${BYPASS_MODE:-False} # True => old_log_prob = rollout_log_prob bypass_loss_type=${BYPASS_LOSS_TYPE:-ppo_clip} # ppo_clip | reinforce rollout_is=${ROLLOUT_IS:-null} # PPO clip already applies the IS ratio rollout_is_threshold=${ROLLOUT_IS_THRESHOLD:-2.0} # single float => TIS upper clamp; "lo_hi" string => IcePop @@ -134,7 +135,7 @@ ray job submit --no-wait --runtime-env $RUNTIME_ENV \ actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ - actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.clip_ratio_c=${clip_ratio_c} \ +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \ actor_rollout_ref.model.use_fused_kernels=True \ actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ From 263a48f57e3dff9bd180c9b8decd3f107f6701a9 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Thu, 23 Jul 2026 15:57:18 +0800 Subject: [PATCH 04/11] update --- tests/uni_agent/agents/test_claude_code_agent.py | 6 ++++-- uni_agent/agents/claude_code/agent.py | 3 --- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/uni_agent/agents/test_claude_code_agent.py b/tests/uni_agent/agents/test_claude_code_agent.py index 73acc902..4e81cd0d 100644 --- a/tests/uni_agent/agents/test_claude_code_agent.py +++ b/tests/uni_agent/agents/test_claude_code_agent.py @@ -131,10 +131,12 @@ def test_run_uses_sandbox_default_workdir(): assert len(sandbox.exec_calls) == 1 assert sandbox.exec_calls[0]["workdir"] is None argv = sandbox.exec_calls[0]["argv"] - assert argv[:4] == ["claude", "--bare", "--no-session-persistence", "-p"] + assert argv[:2] == ["claude", "-p"] assert argv[argv.index("--model") + 1] == "policy" assert argv[argv.index("--permission-mode") + 1] == "bypassPermissions" - assert "--disable-slash-commands" in argv + assert "--bare" not in argv + assert "--no-session-persistence" not in argv + assert "--disable-slash-commands" not in argv assert "--dangerously-skip-permissions" not in argv disallowed_tools = argv[argv.index("--disallowedTools") + 1].split(",") assert set(disallowed_tools) == {"Agent", "Task", "WebFetch", "WebSearch", "AskUserQuestion"} diff --git a/uni_agent/agents/claude_code/agent.py b/uni_agent/agents/claude_code/agent.py index 9afb3aaa..e7b9f22c 100644 --- a/uni_agent/agents/claude_code/agent.py +++ b/uni_agent/agents/claude_code/agent.py @@ -164,15 +164,12 @@ def _claude_argv(self, problem: str, system_prompt: str | None) -> list[str]: raise ValueError("claude_code: set config.model.model_name (the model claude sends)") argv = [ "claude", - "--bare", - "--no-session-persistence", "-p", problem, "--model", model, "--permission-mode", cfg.permission_mode, - "--disable-slash-commands", ] if cfg.disallowed_tools: argv += ["--disallowedTools", ",".join(cfg.disallowed_tools)] From 2ffe342a0369e8f1726eeb01498f57ab14eac498 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Thu, 23 Jul 2026 18:26:23 +0800 Subject: [PATCH 05/11] update --- docs/source/quickstart/rl-training.md | 30 ++++++++++++------- .../training/train_qwen3p5_dense.sh | 13 ++++---- mkdocs.yml | 2 +- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/source/quickstart/rl-training.md b/docs/source/quickstart/rl-training.md index 33c3a69a..a4962bc7 100644 --- a/docs/source/quickstart/rl-training.md +++ b/docs/source/quickstart/rl-training.md @@ -1,6 +1,8 @@ -# Train an Agent with RL +# Run Agent RL Training -This guide demonstrates Agentic RL training for both white-box and black-box agents: +Uni-Agent supports RL training for both white-box and black-box Agents. By integrating with the bundled `verl` module, the same Agent workflow can move seamlessly from inference to training. + +This guide demonstrates: 1. Train `Qwen3-Coder-30B-A3B-Instruct` with the white-box `ReAct Agent`. 2. Train `Qwen3.5-4B` with the black-box `Claude Code` Agent. @@ -51,7 +53,7 @@ The Quickstart provides separate configs for the two Agent types: ```yaml - name: swe_bench sandbox: - provider: vefaas + provider: vefaas # <-- Change to your Sandbox provider. runtime_timeout: 7200 agent: name: react @@ -74,7 +76,7 @@ The Quickstart provides separate configs for the two Agent types: - name: swe_rebench sandbox: - provider: vefaas + provider: vefaas # <-- Change to your Sandbox provider. runtime_timeout: 7200 agent: name: react @@ -101,7 +103,7 @@ The Quickstart provides separate configs for the two Agent types: ```yaml - name: swe_bench sandbox: - provider: vefaas + provider: vefaas # <-- Change to your Sandbox provider. runtime_timeout: 7200 agent: name: claude_code @@ -114,7 +116,7 @@ The Quickstart provides separate configs for the two Agent types: - name: swe_rebench sandbox: - provider: vefaas + provider: vefaas # <-- Change to your Sandbox provider. runtime_timeout: 7200 agent: name: claude_code @@ -188,7 +190,7 @@ This recipe trains `Qwen3-Coder-30B-A3B-Instruct` with the ReAct Task Config. Se DATA_DIR=/path/to/data \ RUNTIME_DIR=/path/to/runtime \ NNODES=8 \ -TP=1 PP=1 CP=4 EP=8 ETP=1 \ +TP=1 PP=2 CP=4 EP=8 ETP=1 \ TASK_CONFIG=examples/quickstart/training/task_config_react.yaml \ EXP_NAME=react_qwen3_coder_30b_dppo_tv \ ADV_ESTIMATOR=rloo \ @@ -203,8 +205,6 @@ ROLLOUT_RS=null \ bash examples/quickstart/training/train_qwen3_moe.sh ``` -This command uses TP1, PP1, CP4, and EP8 to split the 128K Agent context while avoiding tensor- and pipeline-parallel communication. It uses the DPPO-TV settings from the verl reference recipe. To run DPPO-KL instead, set `LOSS_MODE=dppo_kl` and use `CLIP_RATIO_LOW=0.05` and `CLIP_RATIO_HIGH=0.05`. - The default layout is: ```text @@ -245,9 +245,17 @@ This recipe trains `Qwen3.5-4B` with the Claude Code Task Config: DATA_DIR=/path/to/data \ RUNTIME_DIR=/path/to/runtime \ NNODES=8 \ -ADV_ESTIMATOR=rloo \ TASK_CONFIG=examples/quickstart/training/task_config_claude_code.yaml \ -EXP_NAME=claude_code_qwen3_5_4b \ +EXP_NAME=claude_code_qwen3_5_4b_dppo_tv \ +ADV_ESTIMATOR=rloo \ +LOSS_MODE=dppo_tv \ +CLIP_RATIO_LOW=0.15 \ +CLIP_RATIO_HIGH=0.15 \ +CLIP_RATIO_C=10000 \ +LOSS_AGG_MODE=seq-mean-token-sum-norm \ +BYPASS_MODE=False \ +ROLLOUT_IS=null \ +ROLLOUT_RS=null \ bash examples/quickstart/training/train_qwen3p5_dense.sh ``` diff --git a/examples/quickstart/training/train_qwen3p5_dense.sh b/examples/quickstart/training/train_qwen3p5_dense.sh index e4c09d1a..1ba283b2 100644 --- a/examples/quickstart/training/train_qwen3p5_dense.sh +++ b/examples/quickstart/training/train_qwen3p5_dense.sh @@ -35,6 +35,7 @@ kl_loss_coef=${KL_LOSS_COEF:-0.0} clip_ratio_low=${CLIP_RATIO_LOW:-0.2} clip_ratio_high=${CLIP_RATIO_HIGH:-0.28} +clip_ratio_c=${CLIP_RATIO_C:-10.0} # Response length parameters max_prompt_length=${MAX_PROMPT_LENGTH:-$((1024 * 8))} @@ -44,7 +45,7 @@ overlong_buffer_len=${OVERLONG_BUFFER_LEN:-$((1024 * 4))} # unused overlong_penalty_factor=${OVERLONG_PENALTY_FACTOR:-1.0} loss_agg_mode=${LOSS_AGG_MODE:-"token-mean"} -loss_mode=${LOSS_MODE:-bypass_mode} +loss_mode=${LOSS_MODE:-vanilla} # Algorithm temperature=${TEMPERATURE:-1.0} @@ -88,11 +89,11 @@ lr_decay_steps=${LR_DECAY_STEPS:-2000} test_freq=${TEST_FREQ:-10} # ============================================================================ -# Behavior-policy anchored PPO: ratio = pi_train / pi_rollout. -# The PPO ratio is both the importance-sampling ratio and the proximal ratio, -# so explicit rollout IS weights stay disabled to avoid double counting. +# Rollout correction is disabled by default for the standard GRPO + PPO +# baseline. Override these variables to enable behavior-anchor or decoupled +# rollout-correction experiments. # ============================================================================ -bypass_mode=${BYPASS_MODE:-True} # True => old_log_prob = rollout_log_prob +bypass_mode=${BYPASS_MODE:-False} # True => old_log_prob = rollout_log_prob bypass_loss_type=${BYPASS_LOSS_TYPE:-ppo_clip} # ppo_clip | reinforce rollout_is=${ROLLOUT_IS:-null} # PPO clip already applies the IS ratio rollout_is_threshold=${ROLLOUT_IS_THRESHOLD:-2.0} # single float => TIS upper clamp; "lo_hi" string => IcePop @@ -126,7 +127,7 @@ ray job submit --no-wait --runtime-env $RUNTIME_ENV \ actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ - actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.clip_ratio_c=${clip_ratio_c} \ +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \ actor_rollout_ref.model.use_fused_kernels=False \ actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ diff --git a/mkdocs.yml b/mkdocs.yml index f059d122..6a21128d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -52,7 +52,7 @@ nav: - Installation: quickstart/installation.md - Launch a Sandbox: quickstart/launch-sandbox.md - Run Agent Inference: quickstart/agent-inference.md - - Train an Agent with RL: quickstart/rl-training.md + - Run Agent RL Training: quickstart/rl-training.md - Concepts: - Overview: concepts/index.md - Sandbox: concepts/sandbox.md From 5171e39cb45f7eb0d70df83e949e2dc317cfb219 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Thu, 23 Jul 2026 18:36:07 +0800 Subject: [PATCH 06/11] update --- docs/source/quickstart/rl-training.md | 2 + .../training/train_qwen3p5_dense.sh | 1 + .../test_generate_sequences_on_cpu.py | 59 ++++++++++++++++++- uni_agent/framework/framework.py | 43 ++++++++++++++ 4 files changed, 104 insertions(+), 1 deletion(-) diff --git a/docs/source/quickstart/rl-training.md b/docs/source/quickstart/rl-training.md index a4962bc7..4e4c73f0 100644 --- a/docs/source/quickstart/rl-training.md +++ b/docs/source/quickstart/rl-training.md @@ -259,6 +259,8 @@ ROLLOUT_RS=null \ bash examples/quickstart/training/train_qwen3p5_dense.sh ``` +The Claude Code runner sets `trajectory_selection=longest`. If a Gateway session materializes multiple trajectories, the Framework keeps only the trajectory with the most model-generated tokens before reward assignment, artifact dumping, and TransferQueue writes. + The script expects: ```text diff --git a/examples/quickstart/training/train_qwen3p5_dense.sh b/examples/quickstart/training/train_qwen3p5_dense.sh index 1ba283b2..dc1b9a14 100644 --- a/examples/quickstart/training/train_qwen3p5_dense.sh +++ b/examples/quickstart/training/train_qwen3p5_dense.sh @@ -187,6 +187,7 @@ ray job submit --no-wait --runtime-env $RUNTIME_ENV \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_fqn=uni_agent.framework.task_runner.run_task \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.dispatch_mode=ray_task \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.max_concurrent_sessions=${CONCURRENCY} \ + ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.trajectory_selection=longest \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.task_config_path=${TASK_CONFIG} \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.model_name=${SERVED_MODEL_NAME} \ ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.report_reward=True \ diff --git a/tests/uni_agent/framework/test_generate_sequences_on_cpu.py b/tests/uni_agent/framework/test_generate_sequences_on_cpu.py index fad5b3ec..db9210f7 100644 --- a/tests/uni_agent/framework/test_generate_sequences_on_cpu.py +++ b/tests/uni_agent/framework/test_generate_sequences_on_cpu.py @@ -58,6 +58,7 @@ def _inline_runner_config( runner, *, dispatch_mode: str = "inline_async", + trajectory_selection: str | None = None, ) -> dict[str, object]: runner_key = f"runner-{len(_TEST_INLINE_RUNNERS)}" _TEST_INLINE_RUNNERS[runner_key] = runner @@ -66,6 +67,8 @@ def _inline_runner_config( "runner_kwargs": {"runner_key": runner_key}, "dispatch_mode": dispatch_mode, } + if trajectory_selection is not None: + config["trajectory_selection"] = trajectory_selection return config @@ -276,6 +279,7 @@ def _trajectory( *, prompt_ids: list[int] | None = None, response_ids: list[int] | None = None, + response_mask: list[int] | None = None, response_logprobs: list[float] | None = None, reward_info: dict[str, object] | None = None, num_turns: int = 2, @@ -283,10 +287,11 @@ def _trajectory( ): prompt_ids = prompt_ids or [10, 11] response_ids = response_ids or [20, 21] + response_mask = response_mask if response_mask is not None else [1] * len(response_ids) return Trajectory( prompt_ids=prompt_ids, response_ids=response_ids, - response_mask=[1] * len(response_ids), + response_mask=response_mask, response_logprobs=response_logprobs, reward_info=dict(reward_info or {}), reward_score=None, @@ -641,6 +646,58 @@ async def test_generate_sequences_batches_length_trajectory_before_normal_trajec assert batch["fields"]["responses"][1].tolist() == [21] +@pytest.mark.asyncio +async def test_generate_sequences_selects_longest_model_token_trajectory(fake_tq): + runtime = _FakeGatewayManager( + { + "session-sample-0-rollout-0": [ + _trajectory( + response_ids=[20, 21, 22, 23, 24, 25], + response_mask=[1, 0, 0, 0, 0, 0], + num_turns=10, + ), + _trajectory( + response_ids=[30, 31, 32], + response_mask=[1, 1, 1], + num_turns=2, + ), + ] + } + ) + framework = await _build_framework_with_agent_runners( + agent_runners={ + "runner": _inline_runner_config( + _async_noop_runner, + trajectory_selection="longest", + ) + }, + gateway_manager=runtime, + ) + + await framework.generate_sequences(_build_prompts(count=1, global_steps=8)) + + assert len(fake_tq.batch_puts) == 1 + batch = fake_tq.batch_puts[0] + assert batch["keys"] == ["uid-0_0_0"] + assert batch["fields"]["responses"][0].tolist() == [30, 31, 32] + assert batch["fields"]["response_mask"][0].tolist() == [1, 1, 1] + assert batch["fields"]["num_turns"].tolist() == [2] + + +@pytest.mark.asyncio +async def test_framework_rejects_unknown_trajectory_selection(fake_tq): + with pytest.raises(ValueError, match="Unknown trajectory selection"): + await _build_framework_with_agent_runners( + agent_runners={ + "runner": _inline_runner_config( + _async_noop_runner, + trajectory_selection="shortest", + ) + }, + gateway_manager=_FakeGatewayManager({}), + ) + + @pytest.mark.asyncio async def test_generate_sequences_keeps_successful_sessions_when_one_session_fails(fake_tq): """A failed rollout session aborts only that session; other successful diff --git a/uni_agent/framework/framework.py b/uni_agent/framework/framework.py index a9e1610a..988e604a 100644 --- a/uni_agent/framework/framework.py +++ b/uni_agent/framework/framework.py @@ -53,6 +53,7 @@ class _RunnerConfig: runner_kwargs: dict[str, object] dispatch_mode: str max_concurrent_sessions: int + trajectory_selection: str = "all" def __post_init__(self) -> None: if not self.runner_fqn: @@ -61,6 +62,10 @@ def __post_init__(self) -> None: raise ValueError(f"Unknown dispatch mode: {self.dispatch_mode}") if self.max_concurrent_sessions < 0: raise ValueError(f"max_concurrent_sessions must be non-negative, got {self.max_concurrent_sessions}") + if self.trajectory_selection not in {"all", "longest"}: + raise ValueError( + f"Unknown trajectory selection: {self.trajectory_selection}. Expected 'all' or 'longest'" + ) @classmethod def from_config(cls, runner_name: object, runner_cfg) -> _RunnerConfig: @@ -78,12 +83,14 @@ def from_config(cls, runner_name: object, runner_cfg) -> _RunnerConfig: runner_kwargs["tool_config"] = tool_config dispatch_mode = str(runner_cfg.get("dispatch_mode", "inline_async")) max_concurrent_sessions = int(runner_cfg.get("max_concurrent_sessions", 0) or 0) + trajectory_selection = str(runner_cfg.get("trajectory_selection", "all")) try: return cls( runner_fqn="" if runner_fqn is None else str(runner_fqn), runner_kwargs=runner_kwargs, dispatch_mode=dispatch_mode, max_concurrent_sessions=max_concurrent_sessions, + trajectory_selection=trajectory_selection, ) except ValueError as exc: raise ValueError(f"agent_runners.{runner_name}: {exc}") from exc @@ -135,6 +142,37 @@ def _short_failure_reason(error: BaseException) -> str: return f"{error.__class__.__name__}:{message}"[:512] +def _select_session_trajectories( + session_id: str, + trajectories: list[Trajectory], + selection: str, +) -> list[Trajectory]: + """Apply the runner's trajectory-retention policy before scoring and TQ writes.""" + if selection == "all" or len(trajectories) <= 1: + return trajectories + + index, trajectory = max( + enumerate(trajectories), + key=lambda item: ( + sum(item[1].response_mask), + len(item[1].response_ids), + item[1].num_turns, + item[0], + ), + ) + logger.info( + "session %s: selected longest trajectory index=%s model_tokens=%s " + "response_tokens=%s turns=%s candidates=%s", + session_id, + index, + sum(trajectory.response_mask), + len(trajectory.response_ids), + trajectory.num_turns, + len(trajectories), + ) + return [trajectory] + + _TQ_NESTED_SEQUENCE_FIELDS = { "prompts", "responses", @@ -650,6 +688,11 @@ async def _run_session( **({"tools_kwargs": tools_kwargs} if tools_kwargs is not None else {}), ) session_trajectories = await self.gateway_manager.finalize_session(session_id) + session_trajectories = _select_session_trajectories( + session_id, + session_trajectories, + runner_config.trajectory_selection, + ) except Exception: logger.exception("session %s failed (runner=%s); aborting session", session_id, runner_name) await self.gateway_manager.abort_session(session_id) From cfd3b9855ffc4c7f00cd09709d35d64faf9fcc04 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Thu, 23 Jul 2026 18:36:24 +0800 Subject: [PATCH 07/11] update --- uni_agent/framework/framework.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/uni_agent/framework/framework.py b/uni_agent/framework/framework.py index 988e604a..fe39909c 100644 --- a/uni_agent/framework/framework.py +++ b/uni_agent/framework/framework.py @@ -63,9 +63,7 @@ def __post_init__(self) -> None: if self.max_concurrent_sessions < 0: raise ValueError(f"max_concurrent_sessions must be non-negative, got {self.max_concurrent_sessions}") if self.trajectory_selection not in {"all", "longest"}: - raise ValueError( - f"Unknown trajectory selection: {self.trajectory_selection}. Expected 'all' or 'longest'" - ) + raise ValueError(f"Unknown trajectory selection: {self.trajectory_selection}. Expected 'all' or 'longest'") @classmethod def from_config(cls, runner_name: object, runner_cfg) -> _RunnerConfig: @@ -161,8 +159,7 @@ def _select_session_trajectories( ), ) logger.info( - "session %s: selected longest trajectory index=%s model_tokens=%s " - "response_tokens=%s turns=%s candidates=%s", + "session %s: selected longest trajectory index=%s model_tokens=%s response_tokens=%s turns=%s candidates=%s", session_id, index, sum(trajectory.response_mask), From 0e907defd654d7c2278ad50a35436b6a0055c4a3 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Thu, 23 Jul 2026 18:42:37 +0800 Subject: [PATCH 08/11] update --- docs/source/quickstart/rl-training.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/quickstart/rl-training.md b/docs/source/quickstart/rl-training.md index 4e4c73f0..1923d252 100644 --- a/docs/source/quickstart/rl-training.md +++ b/docs/source/quickstart/rl-training.md @@ -190,6 +190,7 @@ This recipe trains `Qwen3-Coder-30B-A3B-Instruct` with the ReAct Task Config. Se DATA_DIR=/path/to/data \ RUNTIME_DIR=/path/to/runtime \ NNODES=8 \ +CONCURRENCY=1024 \ TP=1 PP=2 CP=4 EP=8 ETP=1 \ TASK_CONFIG=examples/quickstart/training/task_config_react.yaml \ EXP_NAME=react_qwen3_coder_30b_dppo_tv \ @@ -245,6 +246,7 @@ This recipe trains `Qwen3.5-4B` with the Claude Code Task Config: DATA_DIR=/path/to/data \ RUNTIME_DIR=/path/to/runtime \ NNODES=8 \ +CONCURRENCY=1024 \ TASK_CONFIG=examples/quickstart/training/task_config_claude_code.yaml \ EXP_NAME=claude_code_qwen3_5_4b_dppo_tv \ ADV_ESTIMATOR=rloo \ From 3ddf26ee071f2b0ec54a4a50e2320f3241ad4523 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Fri, 24 Jul 2026 01:54:58 +0800 Subject: [PATCH 09/11] update --- docs/source/quickstart/rl-training.md | 9 +++++---- .../quickstart/training/task_config_claude_code.yaml | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/source/quickstart/rl-training.md b/docs/source/quickstart/rl-training.md index 1923d252..781b1b40 100644 --- a/docs/source/quickstart/rl-training.md +++ b/docs/source/quickstart/rl-training.md @@ -108,7 +108,7 @@ The Quickstart provides separate configs for the two Agent types: agent: name: claude_code max_turns: 100 - run_timeout: 7200 + run_timeout: 4800 model: temperature: 1.0 top_p: 1.0 @@ -121,7 +121,7 @@ The Quickstart provides separate configs for the two Agent types: agent: name: claude_code max_turns: 100 - run_timeout: 7200 + run_timeout: 4800 model: temperature: 1.0 top_p: 1.0 @@ -245,8 +245,9 @@ This recipe trains `Qwen3.5-4B` with the Claude Code Task Config: ```bash DATA_DIR=/path/to/data \ RUNTIME_DIR=/path/to/runtime \ -NNODES=8 \ +NNODES=4 \ CONCURRENCY=1024 \ +TP=4 PP=2 CP=1 \ TASK_CONFIG=examples/quickstart/training/task_config_claude_code.yaml \ EXP_NAME=claude_code_qwen3_5_4b_dppo_tv \ ADV_ESTIMATOR=rloo \ @@ -261,7 +262,7 @@ ROLLOUT_RS=null \ bash examples/quickstart/training/train_qwen3p5_dense.sh ``` -The Claude Code runner sets `trajectory_selection=longest`. If a Gateway session materializes multiple trajectories, the Framework keeps only the trajectory with the most model-generated tokens before reward assignment, artifact dumping, and TransferQueue writes. +The Claude Code runner sets `trajectory_selection=longest`. If a Gateway session materializes multiple trajectories, the Framework keeps only the trajectory with the most model-generated tokens for RL training. The script expects: diff --git a/examples/quickstart/training/task_config_claude_code.yaml b/examples/quickstart/training/task_config_claude_code.yaml index 7aa08b5f..172a3230 100644 --- a/examples/quickstart/training/task_config_claude_code.yaml +++ b/examples/quickstart/training/task_config_claude_code.yaml @@ -5,7 +5,7 @@ agent: name: claude_code max_turns: 200 - run_timeout: 7200 + run_timeout: 4800 model: temperature: 1.0 top_p: 1.0 @@ -18,7 +18,7 @@ agent: name: claude_code max_turns: 200 - run_timeout: 7200 + run_timeout: 4800 model: temperature: 1.0 top_p: 1.0 From b0f72a5695325fdd5b62f30b9b743c4af889e036 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Fri, 24 Jul 2026 02:24:05 +0800 Subject: [PATCH 10/11] update --- README.md | 2 +- docs/source/benchmark/index.md | 2 +- docs/source/benchmark/rl-training.md | 2 +- examples/agent_train/task_config.yaml | 49 ----- examples/agent_train/train_qwen3_moe.sh | 261 ----------------------- examples/agent_train/train_qwen3p5_4b.sh | 230 -------------------- 6 files changed, 3 insertions(+), 543 deletions(-) delete mode 100644 examples/agent_train/task_config.yaml delete mode 100644 examples/agent_train/train_qwen3_moe.sh delete mode 100644 examples/agent_train/train_qwen3p5_4b.sh diff --git a/README.md b/README.md index 688061c1..191f311f 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Detailed settings and additional reference results are available in [Inference a ### Agent Reinforcement Learning Uni-Agent supports agent RL training with the same interaction stack used at inference time. We provide fully async training recipes across multiple tasks, models and datasets, with GRPO/GSPO-style objectives and partial rollout support. -Example scripts are available in [examples/agent_train](examples/agent_train). +Example scripts are available in [examples/quickstart/training](examples/quickstart/training). | Model | Dataset | Method | Setting | Base | RL | diff --git a/docs/source/benchmark/index.md b/docs/source/benchmark/index.md index 44fc034e..aad39666 100644 --- a/docs/source/benchmark/index.md +++ b/docs/source/benchmark/index.md @@ -33,4 +33,4 @@ Inference and training evolve quickly. A complete result should record: Inference entry points live under `examples/inference/`. Quickstart Task Configs and Runtime Env examples live under `examples/quickstart/inference/`. -RL recipes live under `examples/agent_train/`. Each published result should link back to a runnable recipe and retain its validation curves and configuration. +RL recipes live under `examples/quickstart/training/`. Each published result should link back to a runnable recipe and retain its validation curves and configuration. diff --git a/docs/source/benchmark/rl-training.md b/docs/source/benchmark/rl-training.md index 3a5b7d74..cd505b21 100644 --- a/docs/source/benchmark/rl-training.md +++ b/docs/source/benchmark/rl-training.md @@ -40,7 +40,7 @@ This run uses SWE-reBench for training and SWE-Bench Verified for validation, wi ## Reproduce -Training recipes live under `examples/agent_train/`. +Training recipes live under `examples/quickstart/training/`. For every published result, retain: diff --git a/examples/agent_train/task_config.yaml b/examples/agent_train/task_config.yaml deleted file mode 100644 index 17c70d66..00000000 --- a/examples/agent_train/task_config.yaml +++ /dev/null @@ -1,49 +0,0 @@ -- name: swe_bench - sandbox: - provider: seed - runtime_timeout: 7200 - sandbox_kwargs: - memory_gb: 8 - agent: - name: react - max_steps: 200 - tools: - - name: stateful_shell - command_timeout: 120 - env_vars: - PAGER: "cat" - GIT_PAGER: "cat" - MANPAGER: "cat" - TQDM_DISABLE: "1" - PIP_PROGRESS_BAR: "off" - - name: str_replace_editor - - name: submit - model: - temperature: 1.0 - top_p: 1.0 - max_total_tokens: 131072 - -- name: swe_rebench - sandbox: - provider: seed - runtime_timeout: 7200 - sandbox_kwargs: - memory_gb: 8 - agent: - name: react - max_steps: 200 - tools: - - name: stateful_shell - command_timeout: 120 - env_vars: - PAGER: "cat" - GIT_PAGER: "cat" - MANPAGER: "cat" - TQDM_DISABLE: "1" - PIP_PROGRESS_BAR: "off" - - name: str_replace_editor - - name: submit - model: - temperature: 1.0 - top_p: 1.0 - max_total_tokens: 131072 diff --git a/examples/agent_train/train_qwen3_moe.sh b/examples/agent_train/train_qwen3_moe.sh deleted file mode 100644 index 683c8bf7..00000000 --- a/examples/agent_train/train_qwen3_moe.sh +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env bash -set -xeuo pipefail - -DATA_DIR=/mnt/hdfs/yyding -RUNTIME_DIR=/mnt/hdfs/yyding -NNODES=8 -GEN_TP=4 -TEST_FREQ=-1 - -project_name=${PROJECT_NAME:-'Uni-Agent-Qwen3-Coder-30B-megatron'} -exp_name=${EXP_NAME:-"$(date +%Y%m%d%H)_exp"} - -MODEL_PATH=${MODEL_PATH:-"${DATA_DIR}/models/Qwen3-Coder-30B-A3B-Instruct"} -TRAIN_FILE=${TRAIN_FILE:-"${DATA_DIR}/data/uni_agent/swe_rebench_filtered_1150.parquet"} -TEST_FILE=${TEST_FILE:-"${DATA_DIR}/data/uni_agent/swe_bench_verified.parquet"} - -RUNTIME_ENV=${RUNTIME_ENV:-"${RUNTIME_DIR}/data/uni_agent/runtime_env.yaml"} -CKPTS_DIR=${CKPTS_DIR:-"${RUNTIME_DIR}/ckpts/${project_name}/${exp_name}"} -AGENT_LOG_DIR=${AGENT_LOG_DIR:-"${RUNTIME_DIR}/logs/${project_name}/${exp_name}"} -# Must be launched from the repository root so Ray packages both `verl/` and `uni_agent/`. -# --- Agent-framework rollout (replaces the swe_agent agent-loop) -------------- -# Run-wide task base (agent + sandbox + sampling), loaded from this YAML by -# uni_agent.framework.task_runner.run_task and deep-merged onto each row's task. -# Same file-path idea as the old agent_loop_config_path; new (task-config) schema. -TASK_CONFIG=${TASK_CONFIG:-"examples/agent_train/task_config.yaml"} -TOOL_PARSER=${TOOL_PARSER:-"qwen3_coder"} # gateway tool-call parser; MUST match the model chat template -GATEWAY_COUNT=${GATEWAY_COUNT:-8} # gateway actors fronting the engine -CONCURRENCY=${CONCURRENCY:-512} # max in-flight rollout sessions (runner cap) -SERVED_MODEL_NAME=${SERVED_MODEL_NAME:-"$(basename "${MODEL_PATH}")"} - -rollout_mode=${ROLLOUT_MODE:-"async"} -rollout_name=${ROLLOUT_NAME:-"vllm"} # sglang or vllm - -# Algorithm parameters -adv_estimator=${ADV_ESTIMATOR:-grpo} - -use_kl_in_reward=${USE_KL_IN_REWARD:-False} -kl_coef=${KL_COEF:-0.0} -use_kl_loss=${USE_KL_LOSS:-False} -kl_loss_coef=${KL_LOSS_COEF:-0.0} - -clip_ratio_low=${CLIP_RATIO_LOW:-0.2} -clip_ratio_high=${CLIP_RATIO_HIGH:-0.28} - -# Response length parameters -max_prompt_length=${MAX_PROMPT_LENGTH:-$((1024 * 8))} -max_response_length=${MAX_RESPONSE_LENGTH:-$((1024 * 128))} -enable_overlong_buffer=${ENABLE_OVERLONG_BUFFER:-False} -overlong_buffer_len=${OVERLONG_BUFFER_LEN:-$((1024 * 4))} # unused -overlong_penalty_factor=${OVERLONG_PENALTY_FACTOR:-1.0} - -loss_agg_mode=${LOSS_AGG_MODE:-"token-mean"} -loss_mode=${LOSS_MODE:-bypass_mode} - -# Algorithm -temperature=${TEMPERATURE:-1.0} -top_p=${TOP_P:-1.0} -top_k=${TOP_K:--1} -val_temperature=${VAL_TEMPERATURE:-1.0} -val_top_p=${VAL_TOP_P:-0.95} -val_top_k=${VAL_TOP_K:--1} - -# Performance Related Parameter -use_dynamic_bsz=${USE_DYNAMIC_BSZ:-True} -offload=${OFFLOAD:-True} -gen_tp=${GEN_TP:-4} -train_tp=${TP:-4} -train_pp=${PP:-2} -train_cp=${CP:-2} -train_ep=${EP:-8} -train_etp=${ETP:-1} -actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / train_cp)) -infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / train_cp)) - -optimizer_offload_fraction=${OFFLOAD_FRACTION:-1.0} - -# install mbridge -# pip3 install git+https://github.com/ISEEKYAN/mbridge -USE_MBRIDGE=${USE_MBRIDGE:-True} -USE_DIST_CKPT=${USE_DIST_CKPT:-False} - -# V1 colocate_async topology. colocate_async colocates actor + rollout on the same -# GPUs (rollout replicas sleep during the train step), so NNODES is the TOTAL node -# count (replaces the old fully-async NNODES_ROLLOUT + NNODES_TRAIN split). -NNODES=${NNODES:-8} -NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} - -# parameter_sync_step defaults to 1 for colocate_async, so train_batch_size -# (prompts/step) only needs to be > 0 (the old async mode used train_batch_size=0). -# num_warmup_batches pre-fills the rollout pipeline before the first train step. -train_prompt_bsz=${TRAIN_PROMPT_BSZ:-64} -n_resp_per_prompt=${N_RESP_PER_PROMPT:-8} -train_prompt_mini_bsz=${PPO_MINI_BATCH_SIZE:-16} -num_warmup_batches=${NUM_WARMUP_BATCHES:-1} -lr_decay_steps=${LR_DECAY_STEPS:-2000} -test_freq=${TEST_FREQ:-10} - -# ============================================================================ -# Behavior-policy anchored PPO: ratio = pi_train / pi_rollout. -# The PPO ratio is both the importance-sampling ratio and the proximal ratio, -# so explicit rollout IS weights stay disabled to avoid double counting. -# ============================================================================ -bypass_mode=${BYPASS_MODE:-True} # True => old_log_prob = rollout_log_prob -bypass_loss_type=${BYPASS_LOSS_TYPE:-ppo_clip} # ppo_clip | reinforce -rollout_is=${ROLLOUT_IS:-null} # PPO clip already applies the IS ratio -rollout_is_threshold=${ROLLOUT_IS_THRESHOLD:-2.0} # single float => TIS upper clamp; "lo_hi" string => IcePop -rollout_is_batch_normalize=${ROLLOUT_IS_BATCH_NORMALIZE:-False} # normalize IS weights to mean=1.0 within a batch -rollout_rs=${ROLLOUT_RS:-null} # no rejection sampling -rollout_rs_threshold=${ROLLOUT_RS_THRESHOLD:-null} - -# ============================================================================ -# 30B MoE Router Replay -# ============================================================================ -router_replay_mode=${ROUTER_REPLAY_MODE:-disabled} # disabled | R2 | R3 -enable_rollout_routing_replay=${ENABLE_ROLLOUT_ROUTING_REPLAY:-False} # required only for R3 - -ray job submit --no-wait --runtime-env $RUNTIME_ENV \ - -- python3 -m verl.trainer.main_ppo \ - --config-name=ppo_megatron_trainer \ - trainer.use_v1=True \ - trainer.v1.trainer_mode=colocate_async \ - trainer.v1.colocate_async.num_warmup_batches=${num_warmup_batches} \ - transfer_queue.enable=True \ - data.train_files="${TRAIN_FILE}" \ - data.val_files="${TEST_FILE}" \ - data.prompt_key=prompt \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.max_prompt_length=${max_prompt_length} \ - data.max_response_length=${max_response_length} \ - data.train_batch_size=${train_prompt_bsz} \ - data.return_raw_chat=True \ - actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ - actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ - algorithm.adv_estimator=${adv_estimator} \ - algorithm.use_kl_in_reward=${use_kl_in_reward} \ - algorithm.kl_ctrl.kl_coef=${kl_coef} \ - actor_rollout_ref.model.path="${MODEL_PATH}" \ - actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ - actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ - actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ - actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ - actor_rollout_ref.actor.clip_ratio_c=10.0 \ - +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.model.use_fused_kernels=True \ - actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.optim.lr_decay_style='constant' \ - actor_rollout_ref.actor.optim.weight_decay=0.1 \ - actor_rollout_ref.actor.optim.lr_decay_steps=${lr_decay_steps} \ - +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=${optimizer_offload_fraction} \ - +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \ - +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \ - +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \ - actor_rollout_ref.actor.megatron.use_mbridge=$USE_MBRIDGE \ - actor_rollout_ref.actor.megatron.use_dist_checkpointing=$USE_DIST_CKPT \ - actor_rollout_ref.actor.megatron.param_offload=${offload} \ - actor_rollout_ref.actor.megatron.grad_offload=${offload} \ - actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ - actor_rollout_ref.actor.megatron.context_parallel_size=${train_cp} \ - actor_rollout_ref.actor.megatron.expert_model_parallel_size=${train_ep} \ - actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=${train_etp} \ - +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.masked_softmax_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.bias_activation_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.bias_dropout_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.deallocate_pipeline_outputs=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.persist_layer_norm=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.moe_token_dispatcher_type="alltoall" \ - +actor_rollout_ref.actor.megatron.override_transformer_config.moe_router_dtype=fp32 \ - +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_method=uniform \ - +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity=full \ - +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_num_layers=1 \ - algorithm.rollout_correction.bypass_mode=${bypass_mode} \ - algorithm.rollout_correction.rollout_is=${rollout_is} \ - algorithm.rollout_correction.rollout_is_threshold=${rollout_is_threshold} \ - algorithm.rollout_correction.rollout_is_batch_normalize=${rollout_is_batch_normalize} \ - algorithm.rollout_correction.rollout_rs=${rollout_rs} \ - algorithm.rollout_correction.rollout_rs_threshold="${rollout_rs_threshold}" \ - algorithm.rollout_correction.loss_type=${bypass_loss_type} \ - ++actor_rollout_ref.actor.policy_loss.rollout_correction.bypass_mode=${bypass_mode} \ - ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is=${rollout_is} \ - ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is_threshold=${rollout_is_threshold} \ - ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_is_batch_normalize=${rollout_is_batch_normalize} \ - ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_rs=${rollout_rs} \ - ++actor_rollout_ref.actor.policy_loss.rollout_correction.rollout_rs_threshold="${rollout_rs_threshold}" \ - ++actor_rollout_ref.actor.policy_loss.rollout_correction.loss_type=${bypass_loss_type} \ - actor_rollout_ref.actor.megatron.router_replay.mode=${router_replay_mode} \ - actor_rollout_ref.rollout.enable_rollout_routing_replay=${enable_rollout_routing_replay} \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ - +actor_rollout_ref.actor.checkpoint.save_contents=['model','hf_model'] \ - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ - actor_rollout_ref.rollout.multi_turn.enable=True \ - actor_rollout_ref.rollout.multi_turn.max_parallel_calls=1 \ - ++actor_rollout_ref.rollout.multi_turn.format=${TOOL_PARSER} \ - actor_rollout_ref.rollout.agent.num_workers=8 \ - ++actor_rollout_ref.rollout.agent.agent_loop_manager_class=uni_agent.framework.entry.AgentFrameworkRolloutAdapter \ - ++actor_rollout_ref.rollout.custom.agent_framework.gateway_count=${GATEWAY_COUNT} \ - ++actor_rollout_ref.rollout.custom.agent_framework.log_dir=${AGENT_LOG_DIR} \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_fqn=uni_agent.framework.task_runner.run_task \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.dispatch_mode=ray_task \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.max_concurrent_sessions=${CONCURRENCY} \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.task_config_path=${TASK_CONFIG} \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.model_name=${SERVED_MODEL_NAME} \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.report_reward=True \ - ++actor_rollout_ref.rollout.custom.agent_framework.use_reward_loop_worker=False \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ - actor_rollout_ref.rollout.prompt_length=${max_prompt_length} \ - actor_rollout_ref.rollout.response_length=${max_response_length} \ - actor_rollout_ref.rollout.enable_chunked_prefill=True \ - actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.rollout.max_model_len=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.rollout.temperature=${temperature} \ - actor_rollout_ref.rollout.top_p=${top_p} \ - actor_rollout_ref.rollout.top_k=${top_k} \ - actor_rollout_ref.rollout.val_kwargs.temperature=${val_temperature} \ - actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ - actor_rollout_ref.rollout.val_kwargs.top_k=${val_top_k} \ - actor_rollout_ref.rollout.val_kwargs.do_sample=True \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.rollout.name=${rollout_name} \ - actor_rollout_ref.rollout.mode=${rollout_mode} \ - actor_rollout_ref.rollout.calculate_log_probs=True \ - actor_rollout_ref.nccl_timeout=9600 \ - actor_rollout_ref.rollout.enforce_eager=False \ - actor_rollout_ref.rollout.free_cache_engine=True \ - actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ - actor_rollout_ref.ref.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \ - actor_rollout_ref.ref.megatron.param_offload=${offload} \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ - actor_rollout_ref.ref.megatron.context_parallel_size=${train_cp} \ - actor_rollout_ref.ref.megatron.expert_model_parallel_size=${train_ep} \ - actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=${train_etp} \ - reward.reward_manager.name=dapo \ - +reward.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ - +reward.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ - +reward.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ - +reward.reward_kwargs.overlong_buffer_cfg.log=False \ - +reward.reward_kwargs.max_resp_len=${max_response_length} \ - trainer.logger=['console','wandb'] \ - trainer.project_name="${project_name}" \ - trainer.experiment_name="${exp_name}" \ - trainer.val_before_train=False \ - trainer.save_freq=10 \ - trainer.total_epochs=10 \ - trainer.resume_mode=auto \ - trainer.log_val_generations=10 \ - trainer.default_local_dir="${CKPTS_DIR}" \ - trainer.nnodes="${NNODES}" \ - trainer.n_gpus_per_node="${NGPUS_PER_NODE}" \ - trainer.test_freq="${test_freq}" diff --git a/examples/agent_train/train_qwen3p5_4b.sh b/examples/agent_train/train_qwen3p5_4b.sh deleted file mode 100644 index 76a0ae5e..00000000 --- a/examples/agent_train/train_qwen3p5_4b.sh +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env bash -set -xeuo pipefail - -# Qwen3.5-4B agentic RL on SWE tasks via the uni_agent framework, running on -# verl's V1 unified trainer in `colocate_async` mode: the actor and an async -# rollout share the same GPUs (rollout replicas sleep during the train step), -# and rollout output flows through TransferQueue. -# -# The framework plugs in through `agent_loop_manager_class`: verl.trainer.main_ppo's -# TaskRunnerV1 honors that key, forces transfer_queue.enable, and calls our adapter's -# `generate_sequences` (fire-and-forget) which posts each session's reward into TQ as -# rm_scores. Because reward.reward_model.enable is False, the trainer's -# reward_loop_worker_handles are non-None, so `_compute_reward_colocate` is skipped and -# our posted rm_scores are used verbatim (no DAPO ground_truth needed). -# -# Launch from the repo root so Ray packages both `verl/` and `uni_agent/`. - -RAY_DATA_HOME=/mnt/hdfs/yyding -NGPUS_PER_NODE=${NGPUS_PER_NODE:-8} -# colocate_async colocates actor + rollout, so every node runs both. This is the -# TOTAL node count (replaces the old fully-async NNODES_ROLLOUT + NNODES_TRAIN split). -NNODES=${NNODES:-4} - -project_name='Uni-Agent-Qwen3p5-4B' -exp_name=$(date +%Y%m%d%H)_exp - -RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} -MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3.5-4B"} -CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} -AGENT_LOG_DIR=${AGENT_LOG_DIR:-"${RAY_DATA_HOME}/logs/${project_name}/${exp_name}"} -TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/uni_agent/swe_rebench_filtered_1150.parquet"} -TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/uni_agent/swe_bench_verified.parquet"} -RUNTIME_ENV=${RUNTIME_ENV:-"${RAY_DATA_HOME}/data/uni_agent/runtime_env.yaml"} - -# --- Agent-framework rollout (replaces agent_loop_config_path) ---------------- -# Run-wide task base (agent + sandbox + sampling), loaded from this YAML by -# uni_agent.framework.task_runner.run_task and deep-merged onto each data row's task. -TASK_CONFIG=${TASK_CONFIG:-"examples/agent_train/task_config.yaml"} -TOOL_PARSER=${TOOL_PARSER:-"qwen3_coder"} -GATEWAY_COUNT=${GATEWAY_COUNT:-8} -CONCURRENCY=${CONCURRENCY:-1024} - -rollout_mode="async" -rollout_name="vllm" # sglang or vllm - -# Algorithm parameters -adv_estimator=grpo - -use_kl_in_reward=False -kl_coef=0.0 -use_kl_loss=False -kl_loss_coef=0.0 - -clip_ratio_low=0.2 -clip_ratio_high=0.28 - -# Response length parameters -max_prompt_length=$((1024 * 8)) -max_response_length=$((1024 * 128)) -enable_overlong_buffer=False -overlong_buffer_len=$((1024 * 4)) # unused -overlong_penalty_factor=1.0 - -loss_agg_mode="token-mean" -loss_mode=vanilla - -# Algorithm -temperature=1.0 -top_p=1.0 -top_k=-1 -val_temperature=1.0 -val_top_p=0.95 -val_top_k=-1 - -# Performance Related Parameter -use_dynamic_bsz=True -offload=False -gen_tp=2 -train_tp=4 -train_pp=1 -train_cp=2 -train_ep=1 -train_etp=1 -actor_ppo_max_token_len=$(((max_prompt_length + max_response_length) / train_cp)) -infer_ppo_max_token_len=$(((max_prompt_length + max_response_length) / train_cp)) - -optimizer_offload_fraction=1.0 - -# install mbridge -# pip3 install git+https://github.com/ISEEKYAN/mbridge -USE_MBRIDGE=True -USE_DIST_CKPT=False - -# V1 colocate_async batching. parameter_sync_step defaults to 1 for colocate_async, -# so train_batch_size (prompts/step) only needs to be > 0 (unlike the old async mode -# which used train_batch_size=0 + gen_batch_size=1). num_warmup_batches pre-fills the -# rollout pipeline before the first train step for actor/rollout overlap. -train_prompt_bsz=${train_prompt_bsz:-32} -n_resp_per_prompt=8 -train_prompt_mini_bsz=16 -num_warmup_batches=${num_warmup_batches:-1} -lr_decay_steps=${lr_decay_steps:-2000} -test_freq=10 -save_freq=10 -total_epochs=20 - -ray job submit --no-wait --runtime-env $RUNTIME_ENV \ - -- python3 -m verl.trainer.main_ppo \ - --config-name=ppo_megatron_trainer \ - trainer.use_v1=True \ - trainer.v1.trainer_mode=colocate_async \ - trainer.v1.colocate_async.num_warmup_batches=${num_warmup_batches} \ - transfer_queue.enable=True \ - data.train_files="${TRAIN_FILE}" \ - data.val_files="${TEST_FILE}" \ - data.prompt_key=prompt \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.max_prompt_length=${max_prompt_length} \ - data.max_response_length=${max_response_length} \ - data.train_batch_size=${train_prompt_bsz} \ - data.return_raw_chat=True \ - actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ - actor_rollout_ref.actor.policy_loss.loss_mode=${loss_mode} \ - algorithm.adv_estimator=${adv_estimator} \ - algorithm.use_kl_in_reward=${use_kl_in_reward} \ - algorithm.kl_ctrl.kl_coef=${kl_coef} \ - actor_rollout_ref.model.path="${MODEL_PATH}" \ - actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ - actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ - actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ - actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ - actor_rollout_ref.actor.clip_ratio_c=10.0 \ - +actor_rollout_ref.model.override_config.model_config.max_position_embeddings=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.model.use_fused_kernels=False \ - actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.optim.lr_decay_style='constant' \ - actor_rollout_ref.actor.optim.weight_decay=0.1 \ - actor_rollout_ref.actor.optim.lr_decay_steps=${lr_decay_steps} \ - +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_offload_fraction=${optimizer_offload_fraction} \ - +actor_rollout_ref.actor.optim.override_optimizer_config.overlap_cpu_optimizer_d2h_h2d=True \ - +actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=True \ - +actor_rollout_ref.actor.optim.override_optimizer_config.optimizer_cpu_offload=True \ - actor_rollout_ref.actor.megatron.use_mbridge=$USE_MBRIDGE \ - actor_rollout_ref.actor.megatron.use_dist_checkpointing=$USE_DIST_CKPT \ - actor_rollout_ref.actor.megatron.param_offload=${offload} \ - actor_rollout_ref.actor.megatron.grad_offload=${offload} \ - actor_rollout_ref.actor.megatron.optimizer_offload=${offload} \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=${train_tp} \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=${train_pp} \ - actor_rollout_ref.actor.megatron.context_parallel_size=${train_cp} \ - actor_rollout_ref.actor.megatron.expert_model_parallel_size=${train_ep} \ - actor_rollout_ref.actor.megatron.expert_tensor_parallel_size=${train_etp} \ - +actor_rollout_ref.actor.megatron.override_transformer_config.apply_rope_fusion=False \ - +actor_rollout_ref.actor.megatron.override_transformer_config.masked_softmax_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.bias_activation_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.bias_dropout_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.gradient_accumulation_fusion=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.deallocate_pipeline_outputs=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.persist_layer_norm=True \ - +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_method=uniform \ - +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_granularity=full \ - +actor_rollout_ref.actor.megatron.override_transformer_config.recompute_num_layers=1 \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ - +actor_rollout_ref.actor.checkpoint.save_contents=['hf_model'] \ - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ - actor_rollout_ref.rollout.multi_turn.enable=True \ - actor_rollout_ref.rollout.multi_turn.max_parallel_calls=1 \ - ++actor_rollout_ref.rollout.multi_turn.format=${TOOL_PARSER} \ - ++actor_rollout_ref.rollout.agent.agent_loop_manager_class=uni_agent.framework.entry.AgentFrameworkRolloutAdapter \ - ++actor_rollout_ref.rollout.custom.agent_framework.gateway_count=${GATEWAY_COUNT} \ - ++actor_rollout_ref.rollout.custom.agent_framework.log_dir=${AGENT_LOG_DIR} \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_fqn=uni_agent.framework.task_runner.run_task \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.dispatch_mode=inline_async \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.max_concurrent_sessions=${CONCURRENCY} \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.task_config_path=${TASK_CONFIG} \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.model_name=${MODEL_PATH} \ - ++actor_rollout_ref.rollout.custom.agent_framework.agent_runners.task.runner_kwargs.report_reward=True \ - ++actor_rollout_ref.rollout.custom.agent_framework.use_reward_loop_worker=False \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ - actor_rollout_ref.rollout.prompt_length=${max_prompt_length} \ - actor_rollout_ref.rollout.response_length=${max_response_length} \ - actor_rollout_ref.rollout.enable_chunked_prefill=True \ - actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.rollout.max_model_len=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.rollout.temperature=${temperature} \ - actor_rollout_ref.rollout.top_p=${top_p} \ - actor_rollout_ref.rollout.top_k=${top_k} \ - actor_rollout_ref.rollout.val_kwargs.temperature=${val_temperature} \ - actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ - actor_rollout_ref.rollout.val_kwargs.top_k=${val_top_k} \ - actor_rollout_ref.rollout.val_kwargs.do_sample=True \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.rollout.name=${rollout_name} \ - actor_rollout_ref.rollout.mode=${rollout_mode} \ - actor_rollout_ref.rollout.calculate_log_probs=True \ - actor_rollout_ref.nccl_timeout=9600 \ - actor_rollout_ref.rollout.enforce_eager=False \ - actor_rollout_ref.rollout.free_cache_engine=True \ - actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ - actor_rollout_ref.ref.megatron.use_dist_checkpointing=${USE_DIST_CKPT} \ - actor_rollout_ref.ref.megatron.param_offload=${offload} \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=${train_tp} \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=${train_pp} \ - actor_rollout_ref.ref.megatron.context_parallel_size=${train_cp} \ - actor_rollout_ref.ref.megatron.expert_model_parallel_size=${train_ep} \ - actor_rollout_ref.ref.megatron.expert_tensor_parallel_size=${train_etp} \ - reward.reward_manager.name=dapo \ - +reward.reward_kwargs.overlong_buffer_cfg.enable=${enable_overlong_buffer} \ - +reward.reward_kwargs.overlong_buffer_cfg.len=${overlong_buffer_len} \ - +reward.reward_kwargs.overlong_buffer_cfg.penalty_factor=${overlong_penalty_factor} \ - +reward.reward_kwargs.overlong_buffer_cfg.log=False \ - +reward.reward_kwargs.max_resp_len=${max_response_length} \ - trainer.logger=['console','wandb'] \ - trainer.project_name="${project_name}" \ - trainer.experiment_name="${exp_name}" \ - trainer.val_before_train=False \ - trainer.save_freq=${save_freq} \ - trainer.total_epochs=${total_epochs} \ - trainer.resume_mode=auto \ - trainer.log_val_generations=10 \ - trainer.default_local_dir="${CKPTS_DIR}" \ - trainer.nnodes=${NNODES} \ - trainer.n_gpus_per_node=${NGPUS_PER_NODE} \ - trainer.test_freq=${test_freq} From 36dcee478b8d478d2834623f7221da73aad396e9 Mon Sep 17 00:00:00 2001 From: yuyangding Date: Fri, 24 Jul 2026 17:09:39 +0800 Subject: [PATCH 11/11] update --- docs/source/quickstart/rl-training.md | 24 ++++++++++----- .../test_generate_sequences_on_cpu.py | 30 ++++++++++++++++++- uni_agent/framework/framework.py | 13 ++------ 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/docs/source/quickstart/rl-training.md b/docs/source/quickstart/rl-training.md index 781b1b40..821ba6d4 100644 --- a/docs/source/quickstart/rl-training.md +++ b/docs/source/quickstart/rl-training.md @@ -191,18 +191,28 @@ DATA_DIR=/path/to/data \ RUNTIME_DIR=/path/to/runtime \ NNODES=8 \ CONCURRENCY=1024 \ +GEN_TP=4 \ TP=1 PP=2 CP=4 EP=8 ETP=1 \ +TRAIN_PROMPT_BSZ=64 \ +N_RESP_PER_PROMPT=8 \ +PPO_MINI_BATCH_SIZE=16 \ TASK_CONFIG=examples/quickstart/training/task_config_react.yaml \ -EXP_NAME=react_qwen3_coder_30b_dppo_tv \ +EXP_NAME=react_qwen3_coder_30b_gspo_r3 \ ADV_ESTIMATOR=rloo \ -LOSS_MODE=dppo_tv \ -CLIP_RATIO_LOW=0.15 \ -CLIP_RATIO_HIGH=0.15 \ -CLIP_RATIO_C=10000 \ -LOSS_AGG_MODE=seq-mean-token-sum-norm \ +LOSS_MODE=gspo \ +CLIP_RATIO_LOW=4e-4 \ +CLIP_RATIO_HIGH=4e-4 \ +CLIP_RATIO_C=10 \ +LOSS_AGG_MODE=token-mean \ BYPASS_MODE=False \ -ROLLOUT_IS=null \ +ROLLOUT_IS=token \ +ROLLOUT_IS_THRESHOLD=2.0 \ +ROLLOUT_IS_BATCH_NORMALIZE=False \ ROLLOUT_RS=null \ +ROUTER_REPLAY_MODE=R3 \ +ENABLE_ROLLOUT_ROUTING_REPLAY=True \ +LR_DECAY_STEPS=10000 \ +TEST_FREQ=-1 \ bash examples/quickstart/training/train_qwen3_moe.sh ``` diff --git a/tests/uni_agent/framework/test_generate_sequences_on_cpu.py b/tests/uni_agent/framework/test_generate_sequences_on_cpu.py index db9210f7..6b7f66b5 100644 --- a/tests/uni_agent/framework/test_generate_sequences_on_cpu.py +++ b/tests/uni_agent/framework/test_generate_sequences_on_cpu.py @@ -2,10 +2,12 @@ import asyncio +import numpy as np import pytest +import torch from tests.uni_agent.support import logging_runner -from uni_agent.framework.framework import OpenAICompatibleAgentFramework +from uni_agent.framework.framework import OpenAICompatibleAgentFramework, _align_routed_experts from uni_agent.gateway.session import SessionHandle, Trajectory from verl.utils import tensordict_utils as tu @@ -283,6 +285,7 @@ def _trajectory( response_logprobs: list[float] | None = None, reward_info: dict[str, object] | None = None, num_turns: int = 2, + routed_experts: object | None = None, extra_fields: dict[str, object] | None = None, ): prompt_ids = prompt_ids or [10, 11] @@ -296,6 +299,7 @@ def _trajectory( reward_info=dict(reward_info or {}), reward_score=None, num_turns=num_turns, + routed_experts=routed_experts, multi_modal_data={"images": ["raw-image-should-not-be-written"]}, extra_fields=dict(extra_fields or {}), ) @@ -535,6 +539,14 @@ async def test_generate_sequences_writes_tq_schema_for_each_session(monkeypatch, "session-sample-0-rollout-0": [ _trajectory( response_logprobs=[-0.1, -0.2], + routed_experts=np.array( + [ + [[0, 1], [2, 3]], + [[4, 5], [6, 7]], + [[8, 9], [10, 11]], + ], + dtype=np.uint8, + ), extra_fields={"materialization_reason": "max_response_length"}, ) ], @@ -592,6 +604,8 @@ async def test_generate_sequences_writes_tq_schema_for_each_session(monkeypatch, assert fields["input_ids"].is_nested assert fields["response_mask"].is_nested assert fields["position_ids"].is_nested + assert fields["routed_experts"].is_nested + assert fields["routed_experts"].dtype == torch.uint8 assert fields["prompts"][0].tolist() == [10, 11] assert fields["responses"][0].tolist() == [20, 21] assert fields["response_mask"][0].tolist() == [1, 1] @@ -599,6 +613,12 @@ async def test_generate_sequences_writes_tq_schema_for_each_session(monkeypatch, assert fields["input_ids"][0].tolist() == [10, 11, 20, 21] assert fields["attention_mask"][0].tolist() == [1, 1, 1, 1] assert fields["position_ids"][0].tolist() == [0, 1, 2, 3] + assert fields["routed_experts"][0].tolist() == [ + [[0, 1], [2, 3]], + [[4, 5], [6, 7]], + [[8, 9], [10, 11]], + [[0, 0], [0, 0]], + ] assert fields["rollout_log_probs"][0].tolist() == pytest.approx([-0.1, -0.2]) assert fields["rm_scores"][0].tolist() == [0.0, 0.25] assert tu.get(fields, "multi_modal_inputs") == [{}] @@ -615,6 +635,14 @@ async def test_generate_sequences_writes_tq_schema_for_each_session(monkeypatch, assert "multi_modal_data" not in fields.keys() +def test_align_routed_experts_preserves_backend_dtype(): + aligned = _align_routed_experts(np.array([[[256, 511]]], dtype=np.uint16), seq_len=2) + + assert aligned is not None + assert aligned.dtype == torch.uint16 + assert aligned.tolist() == [[[256, 511]], [[0, 0]]] + + @pytest.mark.asyncio async def test_generate_sequences_batches_length_trajectory_before_normal_trajectory(fake_tq): """Keep length metadata in tags when mixed trajectories share one TQ batch.""" diff --git a/uni_agent/framework/framework.py b/uni_agent/framework/framework.py index fe39909c..d7a9b8ed 100644 --- a/uni_agent/framework/framework.py +++ b/uni_agent/framework/framework.py @@ -201,18 +201,11 @@ def _json_default(obj: object) -> object: def _align_routed_experts(source: object, seq_len: int) -> torch.Tensor | None: - """Return R3 routing as an int64 ``[seq_len, layers, topk]`` tensor aligned to input_ids. - - The gateway stores the last turn's routing, which already spans ``prompt + response`` - (the backend re-prefills the full context each turn). Zero-pad / truncate defensively so - the field always matches ``input_ids`` even on early-return trajectories with trailing - context tokens; a wrong length would crash Megatron's packed-sequence replay. - """ - experts = torch.as_tensor(source) + """Return R3 routing as ``[seq_len, layers, topk]`` aligned to input_ids.""" + experts = torch.as_tensor(source, device="cpu") if experts.dim() != 3: return None - experts = experts.to(dtype=torch.int64, device="cpu") - out = torch.zeros((seq_len, experts.shape[1], experts.shape[2]), dtype=torch.int64) + out = torch.zeros((seq_len, experts.shape[1], experts.shape[2]), dtype=experts.dtype) covered = min(experts.shape[0], seq_len) if covered > 0: out[:covered] = experts[:covered]