强化学习训练概述

以下为训练脚本的完整代码，注释中说明了各参数的含义和取值建议：

import asyncio
from dashscope.finetune.reinforcement import (
    RolloutFunctionComponent, RewardFunctionComponent,
    FunctionComponentModel, FunctionComponentRuntime,
    TrainingDataset, ValidationDataset, DataSourceType,
)
from dashscope.finetune.agentic_rl import AgenticRL

async def main():
    client = AgenticRL()

    # client.run() 一步完成：注册函数到 FC → 上传训练数据 → 提交训练任务
    result = await client.run(
        model="qwen3.5-9b",                              # 基座模型
        training_datasets=[TrainingDataset(              # 训练集
            data_source_type=DataSourceType.FILE_ID,
            file_name="./data/calc_train_min.jsonl",
        )],
        validation_datasets=[ValidationDataset(          # 验证集（可选）
            data_source_type=DataSourceType.FILE_ID,
            file_name="./data/calc_validation_min.jsonl",
        )],
        functions=[
            # Rollout 函数：定义模型如何执行策略采样
            RolloutFunctionComponent(
                name="rollout-1",
                timeout=600,
                fcmodel=FunctionComponentModel(
                    classpath="functions.rollout.rollout.CalcXRolloutProcessor"),
                runtime=FunctionComponentRuntime(
                    cpu=2, memory_size=4096, disk_size=512,
                    concurrency=30, capacity=30,
                    min_capacity=30, max_capacity=60,
                    memory_scale_threshold=0.6,
                    concurrency_scale_threshold=0.6,
                    env={})),                            # FC 函数环境变量
            RewardFunctionComponent(
                name="reward-1",
                weight=1.0,
                timeout=120,
                reward_metric_weight={"reward_metric_weightA": 0.3, "reward_metric_weightB": 0.7},
                fcmodel=FunctionComponentModel(
                    classpath="functions.reward.reward.DemoRewardProcessor"),
                runtime=FunctionComponentRuntime(
                    cpu=2, memory_size=4096, disk_size=512,
                    concurrency=30, capacity=30,
                    min_capacity=30, max_capacity=60,
                    memory_scale_threshold=0.6,
                    concurrency_scale_threshold=0.6,
                    env={})),
        ],
        # MTU 计费配置
        resources={
            "charge_type": "mtu_postpaid",   # 后付费
            "mtu_spec_code": "MTU4",         # 训练单元规格
            "mtu_capacity": 24,              # MTU 数量
        },
        # GSPO 算法超参数（以下为 qwen3.5-9b 非 MoE 模型的全部必选超参）
        hyper_parameters={
            "algorithm": "gspo",             # 训练算法
            "batch_size": 64,                # 批大小
            "eval_steps": 1,                 # 每 N 步执行一次验证
            "kl_loss_coef": 0.002,           # KL 散度损失系数
            "learning_rate": 2e-6,           # 学习率
            "lr_scheduler_type": "linear",   # 学习率调度策略
            "max_length": 8192,              # 最大序列长度（prompt + response）
            "n_epochs": 1,                   # 训练轮数
            "n_rollouts": 8,                 # 每个样本的 Rollout 次数
            "ppo_mini_batch_size": 8,        # PPO 小批量大小
            "save_strategy": "epoch",        # 保存策略（steps 或 epoch）
        })

    if result.status_code == 200:
        print(f"训练任务已提交，Job ID: {result.output.job_id}")
    else:
        print(f"提交失败: {result}")

asyncio.run(main())