246 lines
6.6 KiB
JSON
246 lines
6.6 KiB
JSON
|
{
|
||
|
|
"model_name": "DataForge-0.5B-SFT",
|
||
|
|
"model_license": "apache-2.0",
|
||
|
|
"base_model": "Qwen/Qwen2.5-0.5B-Instruct",
|
||
|
|
"teacher_model": "clean-diff-v1",
|
||
|
|
"dataset_repo": "Praneshrajan15/dataforge-sft-trajectories",
|
||
|
|
"dataset_sha": "94e2dd556d4f1260c5123d93ca6bf4f9da9b160a",
|
||
|
|
"dataset_records": 2058,
|
||
|
|
"training_examples": 1958,
|
||
|
|
"heldout_examples": 100,
|
||
|
|
"trajectory_filename": "expert_v4.jsonl",
|
||
|
|
"promotion_slice": "deterministic_normalization",
|
||
|
|
"config_sha256": "5a6d8ea6531da6d1cb2f594e22b7e63ecf190c1db271af5ad143ca0319271ee8",
|
||
|
|
"trajectory_sha256": "11032d48db0b5b5cf1c7b3faa7041f82bdc3115543ec51b8518c682309c8686a",
|
||
|
|
"split_manifest_sha256": "1fd0853f7ca6d0b39dcfcb6edd9a31791eaecebf05712786ea1fda6809d04e2a",
|
||
|
|
"kaggle_hours": 1.147,
|
||
|
|
"base_f1": 0.0,
|
||
|
|
"sft_f1": 0.0077,
|
||
|
|
"base_eval": {
|
||
|
|
"macro_f1": 0.0053,
|
||
|
|
"mean_f1": 0.0054,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0061,
|
||
|
|
"flights": 0.0,
|
||
|
|
"hospital": 0.0099
|
||
|
|
},
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 113,
|
||
|
|
"failure_taxonomy": {
|
||
|
|
"missed_repair": 553,
|
||
|
|
"overrepair": 158,
|
||
|
|
"schema_case_error": 113,
|
||
|
|
"wrong_cell": 40,
|
||
|
|
"wrong_value": 60
|
||
|
|
},
|
||
|
|
"promotion_slice": "deterministic_normalization",
|
||
|
|
"slice_scores": {
|
||
|
|
"deterministic_normalization": {
|
||
|
|
"tasks": 20,
|
||
|
|
"macro_f1": 0.0,
|
||
|
|
"mean_f1": 0.0,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 29,
|
||
|
|
"finish_rate": 0.0,
|
||
|
|
"false_repair_rate": 1.0,
|
||
|
|
"false_repair_count": 73,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"external_reference_required": {
|
||
|
|
"tasks": 66,
|
||
|
|
"macro_f1": 0.0093,
|
||
|
|
"mean_f1": 0.0081,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 70,
|
||
|
|
"finish_rate": 0.0,
|
||
|
|
"false_repair_rate": 0.9882,
|
||
|
|
"false_repair_count": 252,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0154,
|
||
|
|
"flights": 0.0,
|
||
|
|
"hospital": 0.0125
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"not_inferable_from_prompt": {
|
||
|
|
"tasks": 14,
|
||
|
|
"macro_f1": 0.0,
|
||
|
|
"mean_f1": 0.0,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 14,
|
||
|
|
"finish_rate": 0.0,
|
||
|
|
"false_repair_rate": 1.0,
|
||
|
|
"false_repair_count": 46,
|
||
|
|
"dataset_f1": {
|
||
|
|
"flights": 0.0,
|
||
|
|
"hospital": 0.0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"sft_eval": {
|
||
|
|
"macro_f1": 0.0106,
|
||
|
|
"mean_f1": 0.0106,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0107,
|
||
|
|
"flights": 0.0126,
|
||
|
|
"hospital": 0.0084
|
||
|
|
},
|
||
|
|
"parse_success_rate": 0.99,
|
||
|
|
"schema_case_error_count": 16,
|
||
|
|
"failure_taxonomy": {
|
||
|
|
"missed_repair": 519,
|
||
|
|
"overrepair": 249,
|
||
|
|
"schema_case_error": 16,
|
||
|
|
"truncated_json": 1,
|
||
|
|
"wrong_cell": 3,
|
||
|
|
"wrong_value": 92
|
||
|
|
},
|
||
|
|
"promotion_slice": "deterministic_normalization",
|
||
|
|
"slice_scores": {
|
||
|
|
"deterministic_normalization": {
|
||
|
|
"tasks": 20,
|
||
|
|
"macro_f1": 0.0077,
|
||
|
|
"mean_f1": 0.0077,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 0,
|
||
|
|
"finish_rate": 0.05,
|
||
|
|
"false_repair_rate": 0.9867,
|
||
|
|
"false_repair_count": 74,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0077
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"external_reference_required": {
|
||
|
|
"tasks": 66,
|
||
|
|
"macro_f1": 0.014,
|
||
|
|
"mean_f1": 0.0137,
|
||
|
|
"parse_success_rate": 0.9848,
|
||
|
|
"schema_case_error_count": 8,
|
||
|
|
"finish_rate": 0.0758,
|
||
|
|
"false_repair_rate": 0.9831,
|
||
|
|
"false_repair_count": 233,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0154,
|
||
|
|
"flights": 0.016,
|
||
|
|
"hospital": 0.0106
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"not_inferable_from_prompt": {
|
||
|
|
"tasks": 14,
|
||
|
|
"macro_f1": 0.0,
|
||
|
|
"mean_f1": 0.0,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 8,
|
||
|
|
"finish_rate": 0.0,
|
||
|
|
"false_repair_rate": 1.0,
|
||
|
|
"false_repair_count": 53,
|
||
|
|
"dataset_f1": {
|
||
|
|
"flights": 0.0,
|
||
|
|
"hospital": 0.0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"slice_scores": {
|
||
|
|
"base": {
|
||
|
|
"deterministic_normalization": {
|
||
|
|
"tasks": 20,
|
||
|
|
"macro_f1": 0.0,
|
||
|
|
"mean_f1": 0.0,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 29,
|
||
|
|
"finish_rate": 0.0,
|
||
|
|
"false_repair_rate": 1.0,
|
||
|
|
"false_repair_count": 73,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"external_reference_required": {
|
||
|
|
"tasks": 66,
|
||
|
|
"macro_f1": 0.0093,
|
||
|
|
"mean_f1": 0.0081,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 70,
|
||
|
|
"finish_rate": 0.0,
|
||
|
|
"false_repair_rate": 0.9882,
|
||
|
|
"false_repair_count": 252,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0154,
|
||
|
|
"flights": 0.0,
|
||
|
|
"hospital": 0.0125
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"not_inferable_from_prompt": {
|
||
|
|
"tasks": 14,
|
||
|
|
"macro_f1": 0.0,
|
||
|
|
"mean_f1": 0.0,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 14,
|
||
|
|
"finish_rate": 0.0,
|
||
|
|
"false_repair_rate": 1.0,
|
||
|
|
"false_repair_count": 46,
|
||
|
|
"dataset_f1": {
|
||
|
|
"flights": 0.0,
|
||
|
|
"hospital": 0.0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"sft": {
|
||
|
|
"deterministic_normalization": {
|
||
|
|
"tasks": 20,
|
||
|
|
"macro_f1": 0.0077,
|
||
|
|
"mean_f1": 0.0077,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 0,
|
||
|
|
"finish_rate": 0.05,
|
||
|
|
"false_repair_rate": 0.9867,
|
||
|
|
"false_repair_count": 74,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0077
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"external_reference_required": {
|
||
|
|
"tasks": 66,
|
||
|
|
"macro_f1": 0.014,
|
||
|
|
"mean_f1": 0.0137,
|
||
|
|
"parse_success_rate": 0.9848,
|
||
|
|
"schema_case_error_count": 8,
|
||
|
|
"finish_rate": 0.0758,
|
||
|
|
"false_repair_rate": 0.9831,
|
||
|
|
"false_repair_count": 233,
|
||
|
|
"dataset_f1": {
|
||
|
|
"beers": 0.0154,
|
||
|
|
"flights": 0.016,
|
||
|
|
"hospital": 0.0106
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"not_inferable_from_prompt": {
|
||
|
|
"tasks": 14,
|
||
|
|
"macro_f1": 0.0,
|
||
|
|
"mean_f1": 0.0,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 8,
|
||
|
|
"finish_rate": 0.0,
|
||
|
|
"false_repair_rate": 1.0,
|
||
|
|
"false_repair_count": 53,
|
||
|
|
"dataset_f1": {
|
||
|
|
"flights": 0.0,
|
||
|
|
"hospital": 0.0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"evaluation_chunk_width": 4,
|
||
|
|
"evaluation_max_new_tokens": 1024,
|
||
|
|
"parse_success_rate": 1.0,
|
||
|
|
"schema_case_error_count": 0,
|
||
|
|
"quality_gate_failures": [],
|
||
|
|
"prompt_contract_drift": false,
|
||
|
|
"heldout_leakage_detected": false,
|
||
|
|
"quality_milestone": true,
|
||
|
|
"release_status": "quality_improved_verified",
|
||
|
|
"repo_id": "Praneshrajan15/DataForge-0.5B-SFT"
|
||
|
|
}
|