初始化项目,由ModelHub XC社区提供模型
Model: NotoriousH2/Qwen3-4B-Calendar-Agent-RLVR Source: Original Platform
This commit is contained in:
92
evaluation.json
Normal file
92
evaluation.json
Normal file
@@ -0,0 +1,92 @@
|
||||
{
|
||||
"task": "calendar_tool_calling",
|
||||
"timezone": "Asia/Seoul",
|
||||
"samples": 400,
|
||||
"models": [
|
||||
{
|
||||
"model": "Qwen3-4B Base",
|
||||
"valid_tool_call": 1.0,
|
||||
"policy_compliance": 0.8,
|
||||
"task_success": 0.2,
|
||||
"average_tool_calls": 0.8
|
||||
},
|
||||
{
|
||||
"model": "Calendar Agent SFT",
|
||||
"valid_tool_call": 1.0,
|
||||
"policy_compliance": 0.8925,
|
||||
"task_success": 0.89,
|
||||
"average_tool_calls": 2.5525
|
||||
},
|
||||
{
|
||||
"model": "Calendar Agent RLVR",
|
||||
"valid_tool_call": 1.0,
|
||||
"policy_compliance": 0.9925,
|
||||
"task_success": 0.99,
|
||||
"average_tool_calls": 2.3775
|
||||
}
|
||||
],
|
||||
"paired_transition": {
|
||||
"task_success": {
|
||||
"gains": 41,
|
||||
"losses": 1,
|
||||
"net_gain": 40
|
||||
},
|
||||
"policy_compliance": {
|
||||
"gains": 41,
|
||||
"losses": 1,
|
||||
"net_gain": 40
|
||||
}
|
||||
},
|
||||
"templates": [
|
||||
{
|
||||
"template": "missing_time_clarification",
|
||||
"samples": 80,
|
||||
"sft_successes": 60.0,
|
||||
"rlvr_successes": 80.0
|
||||
},
|
||||
{
|
||||
"template": "delete_confirmation",
|
||||
"samples": 80,
|
||||
"sft_successes": 59.0,
|
||||
"rlvr_successes": 80.0
|
||||
},
|
||||
{
|
||||
"template": "conflict_then_create",
|
||||
"samples": 80,
|
||||
"sft_successes": 77.0,
|
||||
"rlvr_successes": 76.0
|
||||
},
|
||||
{
|
||||
"template": "confirmed_delete",
|
||||
"samples": 80,
|
||||
"sft_successes": 80.0,
|
||||
"rlvr_successes": 80.0
|
||||
},
|
||||
{
|
||||
"template": "multi_attendee_create",
|
||||
"samples": 80,
|
||||
"sft_successes": 80.0,
|
||||
"rlvr_successes": 80.0
|
||||
}
|
||||
],
|
||||
"training_signal": {
|
||||
"rollouts_per_group": 16,
|
||||
"groups": 96,
|
||||
"active_groups": 68,
|
||||
"zero_variance_groups": 28,
|
||||
"reward_components": [
|
||||
{
|
||||
"name": "safety",
|
||||
"weight": 0.25
|
||||
},
|
||||
{
|
||||
"name": "policy_progress",
|
||||
"weight": 0.35
|
||||
},
|
||||
{
|
||||
"name": "semantic_final_success",
|
||||
"weight": 0.4
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user