{ "budget_guard": true, "cohens_d": -0.4012815565141226, "delta": -0.019029999999999984, "episodes": [ { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.6307, "budget_score": 0.0727, "latency_score": 0.4393, "overall_score": 0.5549, "sla_score": 0.9, "success_score": 0.65 }, "grader_score": 0.5549, "metrics": { "average_latency_ms": 280.34, "queue_overflow_events": 0, "sla_met": false, "success_rate": 0.65, "total_cost_spent": 1.02, "total_reward": -5.7566 }, "total_reward": -5.756610262130129 }, "seed": 300, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load" ], "episode_length": 20, "grader": { "adaptation_score": 0.5208, "budget_score": 0.3182, "latency_score": 0.4994, "overall_score": 0.5368, "sla_score": 1.0, "success_score": 0.45 }, "grader_score": 0.5368, "metrics": { "average_latency_ms": 250.32, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.6, "total_cost_spent": 0.75, "total_reward": -8.9091 }, "parse_failures": 0, "total_reward": -8.909090909090912 } }, { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.5852, "budget_score": 0.1364, "latency_score": 0.4182, "overall_score": 0.5361, "sla_score": 0.9, "success_score": 0.6 }, "grader_score": 0.5361, "metrics": { "average_latency_ms": 290.91, "queue_overflow_events": 0, "sla_met": false, "success_rate": 0.6, "total_cost_spent": 0.95, "total_reward": -8.4167 }, "total_reward": -8.41672666635194 }, "seed": 301, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b" ], "episode_length": 20, "grader": { "adaptation_score": 0.55, "budget_score": 0.2727, "latency_score": 0.5036, "overall_score": 0.5516, "sla_score": 1.0, "success_score": 0.5 }, "grader_score": 0.5516, "metrics": { "average_latency_ms": 248.19, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.625, "total_cost_spent": 0.8, "total_reward": -7.6364 }, "parse_failures": 0, "total_reward": -7.6363636363636385 } }, { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.7333, "budget_score": 0.0364, "latency_score": 0.5096, "overall_score": 0.614, "sla_score": 1.0, "success_score": 0.7 }, "grader_score": 0.614, "metrics": { "average_latency_ms": 245.22, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.7, "total_cost_spent": 1.06, "total_reward": -2.8182 }, "total_reward": -2.81818181818182 }, "seed": 302, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "shed_load" ], "episode_length": 20, "grader": { "adaptation_score": 0.6944, "budget_score": 0.2727, "latency_score": 0.5664, "overall_score": 0.6081, "sla_score": 1.0, "success_score": 0.55 }, "grader_score": 0.6081, "metrics": { "average_latency_ms": 216.8, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.6875, "total_cost_spent": 0.8, "total_reward": -4.6364 }, "parse_failures": 0, "total_reward": -4.636363636363638 } }, { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.7619, "budget_score": 0.0364, "latency_score": 0.5229, "overall_score": 0.5836, "sla_score": 0.9412, "success_score": 0.6 }, "grader_score": 0.5836, "metrics": { "average_latency_ms": 238.56, "queue_overflow_events": 0, "sla_met": false, "success_rate": 0.7059, "total_cost_spent": 1.06, "total_reward": -4.4089 }, "total_reward": -4.408894533537559 }, "seed": 303, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b" ], "episode_length": 20, "grader": { "adaptation_score": 0.4881, "budget_score": 0.3636, "latency_score": 0.3641, "overall_score": 0.4543, "sla_score": 0.9286, "success_score": 0.3 }, "grader_score": 0.4543, "metrics": { "average_latency_ms": 317.95, "queue_overflow_events": 0, "sla_met": false, "success_rate": 0.4286, "total_cost_spent": 0.7, "total_reward": -16.2086 }, "parse_failures": 0, "total_reward": -16.208615567898335 } }, { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.6607, "budget_score": 0.0818, "latency_score": 0.5069, "overall_score": 0.6058, "sla_score": 1.0, "success_score": 0.7 }, "grader_score": 0.6058, "metrics": { "average_latency_ms": 246.53, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.7, "total_cost_spent": 1.01, "total_reward": -2.5909 }, "total_reward": -2.590909090909092 }, "seed": 304, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load" ], "episode_length": 20, "grader": { "adaptation_score": 0.6696, "budget_score": 0.2727, "latency_score": 0.536, "overall_score": 0.597, "sla_score": 1.0, "success_score": 0.55 }, "grader_score": 0.597, "metrics": { "average_latency_ms": 232.01, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.6875, "total_cost_spent": 0.8, "total_reward": -4.6364 }, "parse_failures": 0, "total_reward": -4.636363636363638 } }, { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.6333, "budget_score": 0.1364, "latency_score": 0.4553, "overall_score": 0.5757, "sla_score": 0.95, "success_score": 0.65 }, "grader_score": 0.5757, "metrics": { "average_latency_ms": 272.37, "queue_overflow_events": 0, "sla_met": false, "success_rate": 0.65, "total_cost_spent": 0.95, "total_reward": -5.3343 }, "total_reward": -5.3342661209730835 }, "seed": 305, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load" ], "episode_length": 20, "grader": { "adaptation_score": 0.5429, "budget_score": 0.1818, "latency_score": 0.4658, "overall_score": 0.5357, "sla_score": 0.9444, "success_score": 0.55 }, "grader_score": 0.5357, "metrics": { "average_latency_ms": 267.08, "queue_overflow_events": 1, "sla_met": false, "success_rate": 0.6111, "total_cost_spent": 0.9, "total_reward": -8.1537 }, "parse_failures": 0, "total_reward": -8.153655932753445 } }, { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.7125, "budget_score": 0.0636, "latency_score": 0.5269, "overall_score": 0.6174, "sla_score": 1.0, "success_score": 0.7 }, "grader_score": 0.6174, "metrics": { "average_latency_ms": 236.53, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.7368, "total_cost_spent": 1.03, "total_reward": -1.1818 }, "total_reward": -1.1818181818181825 }, "seed": 306, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b" ], "episode_length": 20, "grader": { "adaptation_score": 0.5, "budget_score": 0.2727, "latency_score": 0.5516, "overall_score": 0.5662, "sla_score": 1.0, "success_score": 0.55 }, "grader_score": 0.5662, "metrics": { "average_latency_ms": 224.18, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.6875, "total_cost_spent": 0.8, "total_reward": -4.6364 }, "parse_failures": 0, "total_reward": -4.636363636363639 } }, { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.6667, "budget_score": 0.1636, "latency_score": 0.4865, "overall_score": 0.6152, "sla_score": 1.0, "success_score": 0.7 }, "grader_score": 0.6152, "metrics": { "average_latency_ms": 256.73, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.7, "total_cost_spent": 0.92, "total_reward": -2.1818 }, "total_reward": -2.1818181818181825 }, "seed": 307, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load" ], "episode_length": 20, "grader": { "adaptation_score": 0.75, "budget_score": 0.2273, "latency_score": 0.5844, "overall_score": 0.661, "sla_score": 1.0, "success_score": 0.7 }, "grader_score": 0.661, "metrics": { "average_latency_ms": 207.79, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.8235, "total_cost_spent": 0.85, "total_reward": 2.6364 }, "parse_failures": 0, "total_reward": 2.636363636363633 } }, { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.8, "budget_score": 0.0818, "latency_score": 0.53, "overall_score": 0.6233, "sla_score": 1.0, "success_score": 0.65 }, "grader_score": 0.6233, "metrics": { "average_latency_ms": 234.98, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.7222, "total_cost_spent": 1.01, "total_reward": -2.5909 }, "total_reward": -2.590909090909092 }, "seed": 308, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b" ], "episode_length": 20, "grader": { "adaptation_score": 0.7222, "budget_score": 0.2273, "latency_score": 0.536, "overall_score": 0.6157, "sla_score": 1.0, "success_score": 0.6 }, "grader_score": 0.6157, "metrics": { "average_latency_ms": 232.02, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.7059, "total_cost_spent": 0.85, "total_reward": -3.3636 }, "parse_failures": 0, "total_reward": -3.363636363636367 } }, { "heuristic": { "episode_length": 20, "grader": { "adaptation_score": 0.8333, "budget_score": 0.0364, "latency_score": 0.5286, "overall_score": 0.6378, "sla_score": 1.0, "success_score": 0.7 }, "grader_score": 0.6378, "metrics": { "average_latency_ms": 235.72, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.7778, "total_cost_spent": 1.06, "total_reward": 0.1818 }, "total_reward": 0.18181818181817988 }, "seed": 309, "sft": { "actions": [ "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "route_to_b", "route_to_b", "shed_load", "route_to_b", "shed_load", "route_to_b" ], "episode_length": 20, "grader": { "adaptation_score": 0.75, "budget_score": 0.2273, "latency_score": 0.5901, "overall_score": 0.6471, "sla_score": 1.0, "success_score": 0.65 }, "grader_score": 0.6471, "metrics": { "average_latency_ms": 204.93, "queue_overflow_events": 0, "sla_met": true, "success_rate": 0.7647, "total_cost_spent": 0.85, "total_reward": -0.3636 }, "parse_failures": 0, "total_reward": -0.36363636363636687 } } ], "heuristic_ci95": [ 0.5730399213096795, 0.6197200786903205 ], "heuristic_scores": [ 0.5549, 0.5361, 0.614, 0.5836, 0.6058, 0.5757, 0.6174, 0.6152, 0.6233, 0.6378 ], "losses": 7, "mean_heuristic": 0.59638, "mean_sft": 0.57735, "n_seeds": 10, "p_val": 0.8818559338559583, "seeds": [ 300, 301, 302, 303, 304, 305, 306, 307, 308, 309 ], "sft_ci95": [ 0.5333993497309836, 0.6213006502690165 ], "sft_scores": [ 0.5368, 0.5516, 0.6081, 0.4543, 0.597, 0.5357, 0.5662, 0.661, 0.6157, 0.6471 ], "significant": false, "std_heuristic": 0.03262718021663669, "std_sft": 0.061438772593353286, "t_stat": -1.2689637016022048, "task": "hard_multi", "ties": 0, "wins": 3 }