743 lines
18 KiB
JSON
743 lines
18 KiB
JSON
{
|
|
"budget_guard": true,
|
|
"cohens_d": -0.4012815565141226,
|
|
"delta": -0.019029999999999984,
|
|
"episodes": [
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.6307,
|
|
"budget_score": 0.0727,
|
|
"latency_score": 0.4393,
|
|
"overall_score": 0.5549,
|
|
"sla_score": 0.9,
|
|
"success_score": 0.65
|
|
},
|
|
"grader_score": 0.5549,
|
|
"metrics": {
|
|
"average_latency_ms": 280.34,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": false,
|
|
"success_rate": 0.65,
|
|
"total_cost_spent": 1.02,
|
|
"total_reward": -5.7566
|
|
},
|
|
"total_reward": -5.756610262130129
|
|
},
|
|
"seed": 300,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.5208,
|
|
"budget_score": 0.3182,
|
|
"latency_score": 0.4994,
|
|
"overall_score": 0.5368,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.45
|
|
},
|
|
"grader_score": 0.5368,
|
|
"metrics": {
|
|
"average_latency_ms": 250.32,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.6,
|
|
"total_cost_spent": 0.75,
|
|
"total_reward": -8.9091
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": -8.909090909090912
|
|
}
|
|
},
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.5852,
|
|
"budget_score": 0.1364,
|
|
"latency_score": 0.4182,
|
|
"overall_score": 0.5361,
|
|
"sla_score": 0.9,
|
|
"success_score": 0.6
|
|
},
|
|
"grader_score": 0.5361,
|
|
"metrics": {
|
|
"average_latency_ms": 290.91,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": false,
|
|
"success_rate": 0.6,
|
|
"total_cost_spent": 0.95,
|
|
"total_reward": -8.4167
|
|
},
|
|
"total_reward": -8.41672666635194
|
|
},
|
|
"seed": 301,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.55,
|
|
"budget_score": 0.2727,
|
|
"latency_score": 0.5036,
|
|
"overall_score": 0.5516,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.5
|
|
},
|
|
"grader_score": 0.5516,
|
|
"metrics": {
|
|
"average_latency_ms": 248.19,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.625,
|
|
"total_cost_spent": 0.8,
|
|
"total_reward": -7.6364
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": -7.6363636363636385
|
|
}
|
|
},
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.7333,
|
|
"budget_score": 0.0364,
|
|
"latency_score": 0.5096,
|
|
"overall_score": 0.614,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.7
|
|
},
|
|
"grader_score": 0.614,
|
|
"metrics": {
|
|
"average_latency_ms": 245.22,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.7,
|
|
"total_cost_spent": 1.06,
|
|
"total_reward": -2.8182
|
|
},
|
|
"total_reward": -2.81818181818182
|
|
},
|
|
"seed": 302,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.6944,
|
|
"budget_score": 0.2727,
|
|
"latency_score": 0.5664,
|
|
"overall_score": 0.6081,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.55
|
|
},
|
|
"grader_score": 0.6081,
|
|
"metrics": {
|
|
"average_latency_ms": 216.8,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.6875,
|
|
"total_cost_spent": 0.8,
|
|
"total_reward": -4.6364
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": -4.636363636363638
|
|
}
|
|
},
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.7619,
|
|
"budget_score": 0.0364,
|
|
"latency_score": 0.5229,
|
|
"overall_score": 0.5836,
|
|
"sla_score": 0.9412,
|
|
"success_score": 0.6
|
|
},
|
|
"grader_score": 0.5836,
|
|
"metrics": {
|
|
"average_latency_ms": 238.56,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": false,
|
|
"success_rate": 0.7059,
|
|
"total_cost_spent": 1.06,
|
|
"total_reward": -4.4089
|
|
},
|
|
"total_reward": -4.408894533537559
|
|
},
|
|
"seed": 303,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.4881,
|
|
"budget_score": 0.3636,
|
|
"latency_score": 0.3641,
|
|
"overall_score": 0.4543,
|
|
"sla_score": 0.9286,
|
|
"success_score": 0.3
|
|
},
|
|
"grader_score": 0.4543,
|
|
"metrics": {
|
|
"average_latency_ms": 317.95,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": false,
|
|
"success_rate": 0.4286,
|
|
"total_cost_spent": 0.7,
|
|
"total_reward": -16.2086
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": -16.208615567898335
|
|
}
|
|
},
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.6607,
|
|
"budget_score": 0.0818,
|
|
"latency_score": 0.5069,
|
|
"overall_score": 0.6058,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.7
|
|
},
|
|
"grader_score": 0.6058,
|
|
"metrics": {
|
|
"average_latency_ms": 246.53,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.7,
|
|
"total_cost_spent": 1.01,
|
|
"total_reward": -2.5909
|
|
},
|
|
"total_reward": -2.590909090909092
|
|
},
|
|
"seed": 304,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.6696,
|
|
"budget_score": 0.2727,
|
|
"latency_score": 0.536,
|
|
"overall_score": 0.597,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.55
|
|
},
|
|
"grader_score": 0.597,
|
|
"metrics": {
|
|
"average_latency_ms": 232.01,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.6875,
|
|
"total_cost_spent": 0.8,
|
|
"total_reward": -4.6364
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": -4.636363636363638
|
|
}
|
|
},
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.6333,
|
|
"budget_score": 0.1364,
|
|
"latency_score": 0.4553,
|
|
"overall_score": 0.5757,
|
|
"sla_score": 0.95,
|
|
"success_score": 0.65
|
|
},
|
|
"grader_score": 0.5757,
|
|
"metrics": {
|
|
"average_latency_ms": 272.37,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": false,
|
|
"success_rate": 0.65,
|
|
"total_cost_spent": 0.95,
|
|
"total_reward": -5.3343
|
|
},
|
|
"total_reward": -5.3342661209730835
|
|
},
|
|
"seed": 305,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.5429,
|
|
"budget_score": 0.1818,
|
|
"latency_score": 0.4658,
|
|
"overall_score": 0.5357,
|
|
"sla_score": 0.9444,
|
|
"success_score": 0.55
|
|
},
|
|
"grader_score": 0.5357,
|
|
"metrics": {
|
|
"average_latency_ms": 267.08,
|
|
"queue_overflow_events": 1,
|
|
"sla_met": false,
|
|
"success_rate": 0.6111,
|
|
"total_cost_spent": 0.9,
|
|
"total_reward": -8.1537
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": -8.153655932753445
|
|
}
|
|
},
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.7125,
|
|
"budget_score": 0.0636,
|
|
"latency_score": 0.5269,
|
|
"overall_score": 0.6174,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.7
|
|
},
|
|
"grader_score": 0.6174,
|
|
"metrics": {
|
|
"average_latency_ms": 236.53,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.7368,
|
|
"total_cost_spent": 1.03,
|
|
"total_reward": -1.1818
|
|
},
|
|
"total_reward": -1.1818181818181825
|
|
},
|
|
"seed": 306,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.5,
|
|
"budget_score": 0.2727,
|
|
"latency_score": 0.5516,
|
|
"overall_score": 0.5662,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.55
|
|
},
|
|
"grader_score": 0.5662,
|
|
"metrics": {
|
|
"average_latency_ms": 224.18,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.6875,
|
|
"total_cost_spent": 0.8,
|
|
"total_reward": -4.6364
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": -4.636363636363639
|
|
}
|
|
},
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.6667,
|
|
"budget_score": 0.1636,
|
|
"latency_score": 0.4865,
|
|
"overall_score": 0.6152,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.7
|
|
},
|
|
"grader_score": 0.6152,
|
|
"metrics": {
|
|
"average_latency_ms": 256.73,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.7,
|
|
"total_cost_spent": 0.92,
|
|
"total_reward": -2.1818
|
|
},
|
|
"total_reward": -2.1818181818181825
|
|
},
|
|
"seed": 307,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.75,
|
|
"budget_score": 0.2273,
|
|
"latency_score": 0.5844,
|
|
"overall_score": 0.661,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.7
|
|
},
|
|
"grader_score": 0.661,
|
|
"metrics": {
|
|
"average_latency_ms": 207.79,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.8235,
|
|
"total_cost_spent": 0.85,
|
|
"total_reward": 2.6364
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": 2.636363636363633
|
|
}
|
|
},
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.8,
|
|
"budget_score": 0.0818,
|
|
"latency_score": 0.53,
|
|
"overall_score": 0.6233,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.65
|
|
},
|
|
"grader_score": 0.6233,
|
|
"metrics": {
|
|
"average_latency_ms": 234.98,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.7222,
|
|
"total_cost_spent": 1.01,
|
|
"total_reward": -2.5909
|
|
},
|
|
"total_reward": -2.590909090909092
|
|
},
|
|
"seed": 308,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.7222,
|
|
"budget_score": 0.2273,
|
|
"latency_score": 0.536,
|
|
"overall_score": 0.6157,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.6
|
|
},
|
|
"grader_score": 0.6157,
|
|
"metrics": {
|
|
"average_latency_ms": 232.02,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.7059,
|
|
"total_cost_spent": 0.85,
|
|
"total_reward": -3.3636
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": -3.363636363636367
|
|
}
|
|
},
|
|
{
|
|
"heuristic": {
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.8333,
|
|
"budget_score": 0.0364,
|
|
"latency_score": 0.5286,
|
|
"overall_score": 0.6378,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.7
|
|
},
|
|
"grader_score": 0.6378,
|
|
"metrics": {
|
|
"average_latency_ms": 235.72,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.7778,
|
|
"total_cost_spent": 1.06,
|
|
"total_reward": 0.1818
|
|
},
|
|
"total_reward": 0.18181818181817988
|
|
},
|
|
"seed": 309,
|
|
"sft": {
|
|
"actions": [
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b",
|
|
"shed_load",
|
|
"route_to_b"
|
|
],
|
|
"episode_length": 20,
|
|
"grader": {
|
|
"adaptation_score": 0.75,
|
|
"budget_score": 0.2273,
|
|
"latency_score": 0.5901,
|
|
"overall_score": 0.6471,
|
|
"sla_score": 1.0,
|
|
"success_score": 0.65
|
|
},
|
|
"grader_score": 0.6471,
|
|
"metrics": {
|
|
"average_latency_ms": 204.93,
|
|
"queue_overflow_events": 0,
|
|
"sla_met": true,
|
|
"success_rate": 0.7647,
|
|
"total_cost_spent": 0.85,
|
|
"total_reward": -0.3636
|
|
},
|
|
"parse_failures": 0,
|
|
"total_reward": -0.36363636363636687
|
|
}
|
|
}
|
|
],
|
|
"heuristic_ci95": [
|
|
0.5730399213096795,
|
|
0.6197200786903205
|
|
],
|
|
"heuristic_scores": [
|
|
0.5549,
|
|
0.5361,
|
|
0.614,
|
|
0.5836,
|
|
0.6058,
|
|
0.5757,
|
|
0.6174,
|
|
0.6152,
|
|
0.6233,
|
|
0.6378
|
|
],
|
|
"losses": 7,
|
|
"mean_heuristic": 0.59638,
|
|
"mean_sft": 0.57735,
|
|
"n_seeds": 10,
|
|
"p_val": 0.8818559338559583,
|
|
"seeds": [
|
|
300,
|
|
301,
|
|
302,
|
|
303,
|
|
304,
|
|
305,
|
|
306,
|
|
307,
|
|
308,
|
|
309
|
|
],
|
|
"sft_ci95": [
|
|
0.5333993497309836,
|
|
0.6213006502690165
|
|
],
|
|
"sft_scores": [
|
|
0.5368,
|
|
0.5516,
|
|
0.6081,
|
|
0.4543,
|
|
0.597,
|
|
0.5357,
|
|
0.5662,
|
|
0.661,
|
|
0.6157,
|
|
0.6471
|
|
],
|
|
"significant": false,
|
|
"std_heuristic": 0.03262718021663669,
|
|
"std_sft": 0.061438772593353286,
|
|
"t_stat": -1.2689637016022048,
|
|
"task": "hard_multi",
|
|
"ties": 0,
|
|
"wins": 3
|
|
} |