108 lines
2.7 KiB
JSON
108 lines
2.7 KiB
JSON
|
{
|
|||
|
|
"title": "Отчёт об оценке модели Broken_Code_Generation.1.0",
|
|||
|
|
"model": "Broken_Code_Generation.1.0",
|
|||
|
|
"hf_model": "Vilyam888/Broken_Code_Generation.1.0",
|
|||
|
|
"base_model": "Qwen/Qwen2.5-Coder-3B-Instruct",
|
|||
|
|
"evaluation_date": "2026-06-11",
|
|||
|
|
"evaluation_sample": "test.json, N = 100",
|
|||
|
|
"reference_split": "hold-out test",
|
|||
|
|
"generation": {
|
|||
|
|
"temperature": 0.2,
|
|||
|
|
"max_new_tokens": 1200
|
|||
|
|
},
|
|||
|
|
"training": {
|
|||
|
|
"train_loss_final": 0.1867,
|
|||
|
|
"eval_loss_final": 0.2523,
|
|||
|
|
"eval_mean_token_accuracy": 0.9323,
|
|||
|
|
"perplexity_validation": 1.29,
|
|||
|
|
"num_train_epochs": 3,
|
|||
|
|
"global_step": 501,
|
|||
|
|
"eval_by_epoch": [
|
|||
|
|
{
|
|||
|
|
"epoch": 1.0,
|
|||
|
|
"eval_loss": 0.2812,
|
|||
|
|
"eval_mean_token_accuracy": 0.9243,
|
|||
|
|
"perplexity": 1.3247
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"epoch": 2.0,
|
|||
|
|
"eval_loss": 0.2512,
|
|||
|
|
"eval_mean_token_accuracy": 0.9317,
|
|||
|
|
"perplexity": 1.2856
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"epoch": 3.0,
|
|||
|
|
"eval_loss": 0.2523,
|
|||
|
|
"eval_mean_token_accuracy": 0.9323,
|
|||
|
|
"perplexity": 1.2869
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"finetuned_metrics": {
|
|||
|
|
"valid_json_rate": 0.94,
|
|||
|
|
"required_fields_rate": 0.92,
|
|||
|
|
"difficulty_match_rate": 0.96,
|
|||
|
|
"topic_tag_key_match_rate": 0.97,
|
|||
|
|
"bleu4_corpus": 0.68,
|
|||
|
|
"bleu4_title": 0.74,
|
|||
|
|
"bleu4_task_context": 0.66,
|
|||
|
|
"rouge1_f1": 0.73,
|
|||
|
|
"rouge2_f1": 0.58,
|
|||
|
|
"rougeL_f1": 0.71,
|
|||
|
|
"broken_code_syntax_valid_rate": 0.91,
|
|||
|
|
"code_token_f1_broken_code": 0.47,
|
|||
|
|
"codebleu_broken_code": 0.47
|
|||
|
|
},
|
|||
|
|
"baseline_metrics": {
|
|||
|
|
"valid_json_rate": 0.78,
|
|||
|
|
"required_fields_rate": 0.74,
|
|||
|
|
"difficulty_match_rate": 0.85,
|
|||
|
|
"topic_tag_key_match_rate": 0.83,
|
|||
|
|
"bleu4_corpus": 0.52,
|
|||
|
|
"rouge1_f1": 0.57,
|
|||
|
|
"rouge2_f1": 0.41,
|
|||
|
|
"rougeL_f1": 0.54
|
|||
|
|
},
|
|||
|
|
"baseline_vs_finetuned": {
|
|||
|
|
"bleu4_corpus": {
|
|||
|
|
"baseline": 0.52,
|
|||
|
|
"finetuned": 0.68,
|
|||
|
|
"delta": 0.16
|
|||
|
|
},
|
|||
|
|
"difficulty_match_rate": {
|
|||
|
|
"baseline": 0.85,
|
|||
|
|
"finetuned": 0.96,
|
|||
|
|
"delta": 0.11
|
|||
|
|
},
|
|||
|
|
"required_fields_rate": {
|
|||
|
|
"baseline": 0.74,
|
|||
|
|
"finetuned": 0.92,
|
|||
|
|
"delta": 0.18
|
|||
|
|
},
|
|||
|
|
"rouge1_f1": {
|
|||
|
|
"baseline": 0.57,
|
|||
|
|
"finetuned": 0.73,
|
|||
|
|
"delta": 0.16
|
|||
|
|
},
|
|||
|
|
"rouge2_f1": {
|
|||
|
|
"baseline": 0.41,
|
|||
|
|
"finetuned": 0.58,
|
|||
|
|
"delta": 0.17
|
|||
|
|
},
|
|||
|
|
"rougeL_f1": {
|
|||
|
|
"baseline": 0.54,
|
|||
|
|
"finetuned": 0.71,
|
|||
|
|
"delta": 0.17
|
|||
|
|
},
|
|||
|
|
"topic_tag_key_match_rate": {
|
|||
|
|
"baseline": 0.83,
|
|||
|
|
"finetuned": 0.97,
|
|||
|
|
"delta": 0.14
|
|||
|
|
},
|
|||
|
|
"valid_json_rate": {
|
|||
|
|
"baseline": 0.78,
|
|||
|
|
"finetuned": 0.94,
|
|||
|
|
"delta": 0.16
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|