93 lines
1.9 KiB
JSON
93 lines
1.9 KiB
JSON
{
|
|
"task": "calendar_tool_calling",
|
|
"timezone": "Asia/Seoul",
|
|
"samples": 400,
|
|
"models": [
|
|
{
|
|
"model": "Qwen3-4B Base",
|
|
"valid_tool_call": 1.0,
|
|
"policy_compliance": 0.8,
|
|
"task_success": 0.2,
|
|
"average_tool_calls": 0.8
|
|
},
|
|
{
|
|
"model": "Calendar Agent SFT",
|
|
"valid_tool_call": 1.0,
|
|
"policy_compliance": 0.8925,
|
|
"task_success": 0.89,
|
|
"average_tool_calls": 2.5525
|
|
},
|
|
{
|
|
"model": "Calendar Agent RLVR",
|
|
"valid_tool_call": 1.0,
|
|
"policy_compliance": 0.9925,
|
|
"task_success": 0.99,
|
|
"average_tool_calls": 2.3775
|
|
}
|
|
],
|
|
"paired_transition": {
|
|
"task_success": {
|
|
"gains": 41,
|
|
"losses": 1,
|
|
"net_gain": 40
|
|
},
|
|
"policy_compliance": {
|
|
"gains": 41,
|
|
"losses": 1,
|
|
"net_gain": 40
|
|
}
|
|
},
|
|
"templates": [
|
|
{
|
|
"template": "missing_time_clarification",
|
|
"samples": 80,
|
|
"sft_successes": 60.0,
|
|
"rlvr_successes": 80.0
|
|
},
|
|
{
|
|
"template": "delete_confirmation",
|
|
"samples": 80,
|
|
"sft_successes": 59.0,
|
|
"rlvr_successes": 80.0
|
|
},
|
|
{
|
|
"template": "conflict_then_create",
|
|
"samples": 80,
|
|
"sft_successes": 77.0,
|
|
"rlvr_successes": 76.0
|
|
},
|
|
{
|
|
"template": "confirmed_delete",
|
|
"samples": 80,
|
|
"sft_successes": 80.0,
|
|
"rlvr_successes": 80.0
|
|
},
|
|
{
|
|
"template": "multi_attendee_create",
|
|
"samples": 80,
|
|
"sft_successes": 80.0,
|
|
"rlvr_successes": 80.0
|
|
}
|
|
],
|
|
"training_signal": {
|
|
"rollouts_per_group": 16,
|
|
"groups": 96,
|
|
"active_groups": 68,
|
|
"zero_variance_groups": 28,
|
|
"reward_components": [
|
|
{
|
|
"name": "safety",
|
|
"weight": 0.25
|
|
},
|
|
{
|
|
"name": "policy_progress",
|
|
"weight": 0.35
|
|
},
|
|
{
|
|
"name": "semantic_final_success",
|
|
"weight": 0.4
|
|
}
|
|
]
|
|
}
|
|
}
|