# This reference configration yaml is automatically generated via 'scripts/generate_trainer_config.sh'
# in which it invokes 'python3 scripts/print_cfg.py --cfg job --config-name=ppo_megatron_trainer.yaml' to flatten the 'verl/trainer/config/ppo_megatron_trainer.yaml' config fields into a single file.
# Do not modify this file directly.
# The file is usually only for reference and never used.

actor_rollout_ref:
  actor:
    optim:
      _target_: verl.workers.config.McoreOptimizerConfig
      lr: 1.0e-06
      lr_warmup_steps_ratio: 0.0
      total_training_steps: -1
      weight_decay: 0.01
      lr_warmup_steps: -1
      betas:
      - 0.9
      - 0.999
      clip_grad: 1.0
      optimizer: adam
      lr_warmup_init: 0.0
      lr_decay_steps: null
      lr_decay_style: constant
      min_lr: 0.0
      weight_decay_incr_style: constant
      lr_wsd_decay_style: exponential
      lr_wsd_decay_steps: null
      use_checkpoint_opt_param_scheduler: false
      override_optimizer_config: {}
    megatron:
      _target_: verl.workers.config.McoreEngineConfig
      param_offload: false
      grad_offload: false
      optimizer_offload: false
      tensor_model_parallel_size: 1
      expert_model_parallel_size: 1
      expert_tensor_parallel_size: 1
      pipeline_model_parallel_size: 1
      virtual_pipeline_model_parallel_size: null
      context_parallel_size: 1
      sequence_parallel: true
      use_distributed_optimizer: true
      use_dist_checkpointing: false
      dist_checkpointing_path: null
      seed: 42
      override_ddp_config: {}
      override_transformer_config:
        recompute_granularity: null
        recompute_modules:
        - core_attn
        recompute_method: null
        recompute_num_layers: null
        attention_backend: flash
      override_mcore_model_config: {}
      use_mbridge: false
      forward_only: false
    _target_: verl.workers.config.McoreActorConfig
    strategy: megatron
    ppo_mini_batch_size: 256
    ppo_micro_batch_size: null
    ppo_micro_batch_size_per_gpu: null
    use_dynamic_bsz: false
    ppo_max_token_len_per_gpu: 16384
    clip_ratio: 0.2
    clip_ratio_low: 0.2
    clip_ratio_high: 0.2
    freeze_vision_tower: false
    policy_loss:
      _target_: verl.workers.config.PolicyLossConfig
      loss_mode: vanilla
      clip_cov_ratio: 0.0002
      clip_cov_lb: 1.0
      clip_cov_ub: 5.0
      kl_cov_ratio: 0.0002
      ppo_kl_coef: 0.1
    clip_ratio_c: 3.0
    loss_agg_mode: token-mean
    entropy_coeff: 0
    tis_imp_ratio_cap: -1
    use_kl_loss: false
    use_torch_compile: true
    kl_loss_coef: 0.001
    kl_loss_type: low_var_kl
    ppo_epochs: 1
    shuffle: false
    checkpoint:
      _target_: verl.trainer.config.CheckpointConfig
      save_contents:
      - model
      - optimizer
      - extra
      load_contents: ${.save_contents}
      async_save: false
    use_fused_kernels: ${oc.select:actor_rollout_ref.model.use_fused_kernels,false}
    profiler:
      _target_: verl.utils.profiler.ProfilerConfig
      tool: ${oc.select:global_profiler.tool,null}
      enable: false
      all_ranks: false
      ranks: []
      save_path: ${oc.select:global_profiler.save_path,null}
      tool_config:
        nsys:
          _target_: verl.utils.profiler.config.NsightToolConfig
          discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete}
        npu:
          _target_: verl.utils.profiler.config.NPUToolConfig
          contents: []
          level: level1
          analysis: true
          discrete: false
        torch:
          _target_: verl.utils.profiler.config.TorchProfilerToolConfig
          step_start: 0
          step_end: null
        torch_memory:
          _target_: verl.utils.profiler.config.TorchMemoryToolConfig
          trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000}
          stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32}
    data_loader_seed: null
    load_weight: true
  ref:
    strategy: megatron
    use_torch_compile: ${oc.select:actor_rollout_ref.actor.use_torch_compile,true}
    log_prob_micro_batch_size: null
    log_prob_micro_batch_size_per_gpu: null
    log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false}
    log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384}
    profiler:
      _target_: verl.utils.profiler.ProfilerConfig
      tool: ${oc.select:global_profiler.tool,null}
      enable: false
      all_ranks: false
      ranks: []
      save_path: ${oc.select:global_profiler.save_path,null}
      tool_config:
        nsys:
          _target_: verl.utils.profiler.config.NsightToolConfig
          discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete}
        npu:
          _target_: verl.utils.profiler.config.NPUToolConfig
          contents: []
          level: level1
          analysis: true
          discrete: false
        torch:
          _target_: verl.utils.profiler.config.TorchProfilerToolConfig
          step_start: 0
          step_end: null
        torch_memory:
          _target_: verl.utils.profiler.config.TorchMemoryToolConfig
          trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000}
          stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32}
    megatron:
      _target_: verl.workers.config.MegatronEngineConfig
      param_offload: false
      grad_offload: false
      optimizer_offload: false
      tensor_model_parallel_size: 1
      expert_model_parallel_size: 1
      expert_tensor_parallel_size: 1
      pipeline_model_parallel_size: 1
      virtual_pipeline_model_parallel_size: null
      context_parallel_size: 1
      sequence_parallel: true
      use_distributed_optimizer: true
      use_dist_checkpointing: false
      dist_checkpointing_path: null
      seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42}
      override_ddp_config: {}
      override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}}
      override_mcore_model_config: {}
      use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False}
      forward_only: false
    load_weight: true
  rollout:
    _target_: verl.workers.config.RolloutConfig
    name: ???
    mode: sync
    temperature: 1.0
    top_k: -1
    top_p: 1
    prompt_length: ${oc.select:data.max_prompt_length,512}
    response_length: ${oc.select:data.max_response_length,512}
    dtype: bfloat16
    gpu_memory_utilization: 0.5
    ignore_eos: false
    enforce_eager: false
    cudagraph_capture_sizes: null
    free_cache_engine: true
    tensor_model_parallel_size: 2
    max_num_batched_tokens: 8192
    max_model_len: null
    max_num_seqs: 1024
    enable_chunked_prefill: true
    enable_prefix_caching: true
    load_format: dummy
    log_prob_micro_batch_size: null
    log_prob_micro_batch_size_per_gpu: null
    log_prob_use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false}
    log_prob_max_token_len_per_gpu: ${oc.select:actor_rollout_ref.actor.ppo_max_token_len_per_gpu,16384}
    disable_log_stats: true
    do_sample: true
    'n': 1
    over_sample_rate: 0
    multi_stage_wake_up: false
    engine_kwargs:
      vllm: {}
      sglang: {}
    val_kwargs:
      _target_: verl.workers.config.SamplingConfig
      top_k: -1
      top_p: 1.0
      temperature: 0
      'n': 1
      do_sample: false
    multi_turn:
      _target_: verl.workers.config.MultiTurnConfig
      enable: false
      max_assistant_turns: null
      tool_config_path: null
      max_user_turns: null
      max_parallel_calls: 1
      max_tool_response_length: 256
      tool_response_truncate_side: middle
      interaction_config_path: null
      use_inference_chat_template: false
      tokenization_sanity_check_mode: strict
      format: hermes
      num_repeat_rollouts: null
    calculate_log_probs: false
    agent:
      _target_: verl.workers.config.AgentLoopConfig
      num_workers: 8
      agent_loop_config_path: null
      custom_async_server:
        _target_: verl.workers.config.CustomAsyncServerConfig
        path: null
        name: null
    update_weights_bucket_megabytes: 512
    trace:
      _target_: verl.workers.config.TraceConfig
      backend: null
      token2text: false
    skip_rollout: false
    skip_dump_dir: /tmp/rollout_dump
    skip_tokenizer_init: true
    profiler:
      _target_: verl.utils.profiler.ProfilerConfig
      tool: ${oc.select:global_profiler.tool,null}
      enable: ${oc.select:actor_rollout_ref.actor.profiler.enable,false}
      all_ranks: ${oc.select:actor_rollout_ref.actor.profiler.all_ranks,false}
      ranks: ${oc.select:actor_rollout_ref.actor.profiler.ranks,[]}
      save_path: ${oc.select:global_profiler.save_path,null}
      tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null}
    layer_name_map:
      qkv_layer_name: qkv
      gate_proj_layer_name: gate_up
  hybrid_engine: true
  nccl_timeout: 600
  model:
    path: ~/models/deepseek-llm-7b-chat
    custom_chat_template: null
    external_lib: null
    override_config:
      model_config: {}
      moe_config:
        freeze_moe_router: false
    use_fused_kernels: false
    trust_remote_code: false
    use_remove_padding: false
data:
  tokenizer: null
  use_shm: false
  train_files: ~/data/rlhf/gsm8k/train.parquet
  val_files: ~/data/rlhf/gsm8k/test.parquet
  prompt_key: prompt
  reward_fn_key: data_source
  max_prompt_length: 512
  max_response_length: 512
  train_batch_size: 1024
  val_batch_size: null
  return_raw_input_ids: false
  return_raw_chat: false
  return_full_prompt: false
  shuffle: true
  dataloader_num_workers: 8
  validation_shuffle: false
  filter_overlong_prompts: false
  filter_overlong_prompts_workers: 1
  truncation: error
  image_key: images
  video_key: videos
  trust_remote_code: false
  custom_cls:
    path: null
    name: null
  return_multi_modal_inputs: true
  sampler:
    class_path: null
    class_name: null
  datagen:
    path: null
    name: null
  apply_chat_template_kwargs: {}
critic:
  optim:
    _target_: verl.workers.config.McoreOptimizerConfig
    lr: 1.0e-05
    lr_warmup_steps_ratio: 0.0
    total_training_steps: -1
    weight_decay: 0.01
    lr_warmup_steps: -1
    betas:
    - 0.9
    - 0.999
    clip_grad: 1.0
    optimizer: adam
    lr_warmup_init: 0.0
    lr_decay_steps: null
    lr_decay_style: constant
    min_lr: 0.0
    weight_decay_incr_style: constant
    lr_wsd_decay_style: exponential
    lr_wsd_decay_steps: null
    use_checkpoint_opt_param_scheduler: false
    override_optimizer_config: {}
  megatron:
    _target_: verl.workers.config.McoreEngineConfig
    param_offload: false
    grad_offload: false
    optimizer_offload: false
    tensor_model_parallel_size: 1
    expert_model_parallel_size: 1
    expert_tensor_parallel_size: 1
    pipeline_model_parallel_size: 1
    virtual_pipeline_model_parallel_size: null
    context_parallel_size: 1
    sequence_parallel: true
    use_distributed_optimizer: true
    use_dist_checkpointing: false
    dist_checkpointing_path: null
    seed: 42
    override_ddp_config: {}
    override_transformer_config:
      recompute_granularity: null
      recompute_modules:
      - core_attn
      recompute_method: null
      recompute_num_layers: null
      attention_backend: flash
    override_mcore_model_config: {}
    use_mbridge: false
    forward_only: false
  _target_: verl.workers.config.McoreCriticConfig
  rollout_n: ${oc.select:actor_rollout_ref.rollout.n,1}
  strategy: megatron
  enable: null
  model:
    path: ~/models/deepseek-llm-7b-chat
    tokenizer_path: ${oc.select:actor_rollout_ref.model.path,"~/models/deepseek-llm-7b-chat"}
    override_config:
      model_config: {}
      moe_config:
        freeze_moe_router: false
    external_lib: ${oc.select:actor_rollout_ref.model.external_lib,null}
    trust_remote_code: ${oc.select:actor_rollout_ref.model.trust_remote_code,false}
    _target_: verl.trainer.config.BaseModelConfig
  ppo_mini_batch_size: ${oc.select:actor_rollout_ref.actor.ppo_mini_batch_size,256}
  ppo_micro_batch_size: null
  ppo_micro_batch_size_per_gpu: ${oc.select:.ppo_micro_batch_size,null}
  use_dynamic_bsz: ${oc.select:actor_rollout_ref.actor.use_dynamic_bsz,false}
  ppo_max_token_len_per_gpu: 32768
  forward_max_token_len_per_gpu: ${.ppo_max_token_len_per_gpu}
  ppo_epochs: ${oc.select:actor_rollout_ref.actor.ppo_epochs,1}
  shuffle: ${oc.select:actor_rollout_ref.actor.shuffle,false}
  cliprange_value: 0.5
  loss_agg_mode: ${oc.select:actor_rollout_ref.actor.loss_agg_mode,token-mean}
  checkpoint:
    _target_: verl.trainer.config.CheckpointConfig
    save_contents:
    - model
    - optimizer
    - extra
    load_contents: ${.save_contents}
    async_save: false
  profiler:
    _target_: verl.utils.profiler.ProfilerConfig
    tool: ${oc.select:global_profiler.tool,null}
    enable: false
    all_ranks: false
    ranks: []
    save_path: ${oc.select:global_profiler.save_path,null}
    tool_config:
      nsys:
        _target_: verl.utils.profiler.config.NsightToolConfig
        discrete: ${oc.select:global_profiler.global_tool_config.nsys.discrete}
      npu:
        _target_: verl.utils.profiler.config.NPUToolConfig
        contents: []
        level: level1
        analysis: true
        discrete: false
      torch:
        _target_: verl.utils.profiler.config.TorchProfilerToolConfig
        step_start: 0
        step_end: null
      torch_memory:
        _target_: verl.utils.profiler.config.TorchMemoryToolConfig
        trace_alloc_max_entries: ${oc.select:global_profiler.global_tool_config.torch_memory.trace_alloc_max_entries,100000}
        stack_depth: ${oc.select:global_profiler.global_tool_config.torch_memory.stack_depth,32}
  nccl_timeout: 600
  load_weight: true
  data_loader_seed: ${oc.select:actor_rollout_ref.actor.data_loader_seed,null}
reward_model:
  enable: false
  enable_resource_pool: false
  n_gpus_per_node: 0
  nnodes: 0
  strategy: megatron
  model:
    input_tokenizer: ${actor_rollout_ref.model.path}
    path: ~/models/FsfairX-LLaMA3-RM-v0.1
    external_lib: ${actor_rollout_ref.model.external_lib}
    trust_remote_code: false
  micro_batch_size: null
  micro_batch_size_per_gpu: null
  max_length: null
  use_dynamic_bsz: ${critic.use_dynamic_bsz}
  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
  reward_manager: naive
  launch_reward_fn_async: false
  sandbox_fusion:
    url: null
    max_concurrent: 64
    memory_limit_mb: 1024
  profiler:
    _target_: verl.utils.profiler.ProfilerConfig
    tool: ${oc.select:global_profiler.tool,null}
    enable: false
    all_ranks: false
    ranks: []
    save_path: ${oc.select:global_profiler.save_path,null}
    tool_config: ${oc.select:actor_rollout_ref.actor.profiler.tool_config,null}
  nccl_timeout: 600
  megatron:
    _target_: verl.workers.config.MegatronEngineConfig
    param_offload: false
    tensor_model_parallel_size: 1
    expert_model_parallel_size: 1
    expert_tensor_parallel_size: 1
    pipeline_model_parallel_size: 1
    virtual_pipeline_model_parallel_size: null
    context_parallel_size: 1
    sequence_parallel: true
    use_distributed_optimizer: false
    use_dist_checkpointing: false
    dist_checkpointing_path: null
    seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42}
    override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}}
    use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False}
  load_weight: true
custom_reward_function:
  path: null
  name: compute_score
algorithm:
  _target_: verl.trainer.config.AlgoConfig
  gamma: 1.0
  lam: 1.0
  adv_estimator: gae
  norm_adv_by_std_in_grpo: true
  use_kl_in_reward: false
  kl_penalty: kl
  kl_ctrl:
    _target_: verl.trainer.config.KLControlConfig
    type: fixed
    kl_coef: 0.001
    horizon: 10000
    target_kl: 0.1
  use_pf_ppo: false
  pf_ppo:
    reweight_method: pow
    weight_pow: 2.0
trainer:
  balance_batch: true
  total_epochs: 30
  total_training_steps: null
  project_name: verl_examples
  experiment_name: gsm8k
  logger:
  - console
  - wandb
  log_val_generations: 0
  nnodes: 1
  n_gpus_per_node: 8
  save_freq: -1
  esi_redundant_time: 0
  resume_mode: auto
  resume_from_path: null
  del_local_ckpt_after_load: false
  val_before_train: true
  test_freq: -1
  critic_warmup: 0
  default_hdfs_dir: null
  default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name}
  max_actor_ckpt_to_keep: null
  max_critic_ckpt_to_keep: null
  ray_wait_register_center_timeout: 300
  device: cuda
  rollout_data_dir: null
global_profiler:
  _target_: verl.utils.profiler.ProfilerConfig
  tool: null
  steps: null
  profile_continuous_steps: false
  save_path: outputs/profile
  global_tool_config:
    nsys:
      discrete: false
      controller_nsight_options:
        trace: cuda,nvtx,cublas,ucx
        cuda-memory-usage: 'true'
        cuda-graph-trace: graph
      worker_nsight_options:
        trace: cuda,nvtx,cublas,ucx
        cuda-memory-usage: 'true'
        cuda-graph-trace: graph
        capture-range: cudaProfilerApi
        capture-range-end: null
        kill: none
    torch_memory:
      trace_alloc_max_entries: 100000
      stack_depth: 32
      context: all
      stacks: all
      kw_args: {}
ray_kwargs:
  ray_init:
    num_cpus: null
  timeline_json_file: null
