{
  "title": "Отчёт об оценке модели Broken_Code_Generation.1.0",
  "model": "Broken_Code_Generation.1.0",
  "hf_model": "Vilyam888/Broken_Code_Generation.1.0",
  "base_model": "Qwen/Qwen2.5-Coder-3B-Instruct",
  "evaluation_date": "2026-06-11",
  "evaluation_sample": "test.json, N = 100",
  "reference_split": "hold-out test",
  "generation": {
    "temperature": 0.2,
    "max_new_tokens": 1200
  },
  "training": {
    "train_loss_final": 0.1867,
    "eval_loss_final": 0.2523,
    "eval_mean_token_accuracy": 0.9323,
    "perplexity_validation": 1.29,
    "num_train_epochs": 3,
    "global_step": 501,
    "eval_by_epoch": [
      {
        "epoch": 1.0,
        "eval_loss": 0.2812,
        "eval_mean_token_accuracy": 0.9243,
        "perplexity": 1.3247
      },
      {
        "epoch": 2.0,
        "eval_loss": 0.2512,
        "eval_mean_token_accuracy": 0.9317,
        "perplexity": 1.2856
      },
      {
        "epoch": 3.0,
        "eval_loss": 0.2523,
        "eval_mean_token_accuracy": 0.9323,
        "perplexity": 1.2869
      }
    ]
  },
  "finetuned_metrics": {
    "valid_json_rate": 0.94,
    "required_fields_rate": 0.92,
    "difficulty_match_rate": 0.96,
    "topic_tag_key_match_rate": 0.97,
    "bleu4_corpus": 0.68,
    "bleu4_title": 0.74,
    "bleu4_task_context": 0.66,
    "rouge1_f1": 0.73,
    "rouge2_f1": 0.58,
    "rougeL_f1": 0.71,
    "broken_code_syntax_valid_rate": 0.91,
    "code_token_f1_broken_code": 0.47,
    "codebleu_broken_code": 0.47
  },
  "baseline_metrics": {
    "valid_json_rate": 0.78,
    "required_fields_rate": 0.74,
    "difficulty_match_rate": 0.85,
    "topic_tag_key_match_rate": 0.83,
    "bleu4_corpus": 0.52,
    "rouge1_f1": 0.57,
    "rouge2_f1": 0.41,
    "rougeL_f1": 0.54
  },
  "baseline_vs_finetuned": {
    "bleu4_corpus": {
      "baseline": 0.52,
      "finetuned": 0.68,
      "delta": 0.16
    },
    "difficulty_match_rate": {
      "baseline": 0.85,
      "finetuned": 0.96,
      "delta": 0.11
    },
    "required_fields_rate": {
      "baseline": 0.74,
      "finetuned": 0.92,
      "delta": 0.18
    },
    "rouge1_f1": {
      "baseline": 0.57,
      "finetuned": 0.73,
      "delta": 0.16
    },
    "rouge2_f1": {
      "baseline": 0.41,
      "finetuned": 0.58,
      "delta": 0.17
    },
    "rougeL_f1": {
      "baseline": 0.54,
      "finetuned": 0.71,
      "delta": 0.17
    },
    "topic_tag_key_match_rate": {
      "baseline": 0.83,
      "finetuned": 0.97,
      "delta": 0.14
    },
    "valid_json_rate": {
      "baseline": 0.78,
      "finetuned": 0.94,
      "delta": 0.16
    }
  }
}