初始化项目，由ModelHub XC社区提供模型

Model: Vilyam888/Broken_Code_Generation.1.0 Source: Original Platform
2026-06-16 06:33:19 +08:00
commit b0af1ebc67
22 changed files with 1377 additions and 0 deletions
--- a/metrics/01_training_perplexity.json
+++ b/metrics/01_training_perplexity.json
@@ -0,0 +1,39 @@
+{
+  "metric_group": "training_perplexity",
+  "model": "Broken_Code_Generation.1.0",
+  "hf_model": "Vilyam888/Broken_Code_Generation.1.0",
+  "base_model": "Qwen/Qwen2.5-Coder-3B-Instruct",
+  "adapter_dir": "outputs/qwen25-coder-3b-qlora",
+  "checkpoint": "outputs/qwen25-coder-3b-qlora/checkpoint-501",
+  "source": "outputs/qwen25-coder-3b-qlora/checkpoint-501/trainer_state.json",
+  "validation_file": "prepared_data/val.json",
+  "evaluation_date": "2026-06-11",
+  "metrics": {
+    "train_loss_final": 0.1867,
+    "eval_loss_final": 0.2523,
+    "eval_mean_token_accuracy": 0.9323,
+    "perplexity_validation": 1.29,
+    "num_train_epochs": 3,
+    "global_step": 501,
+    "eval_by_epoch": [
+      {
+        "epoch": 1.0,
+        "eval_loss": 0.2812,
+        "eval_mean_token_accuracy": 0.9243,
+        "perplexity": 1.3247
+      },
+      {
+        "epoch": 2.0,
+        "eval_loss": 0.2512,
+        "eval_mean_token_accuracy": 0.9317,
+        "perplexity": 1.2856
+      },
+      {
+        "epoch": 3.0,
+        "eval_loss": 0.2523,
+        "eval_mean_token_accuracy": 0.9323,
+        "perplexity": 1.2869
+      }
+    ]
+  }
+}
--- a/metrics/02_json_validity.json
+++ b/metrics/02_json_validity.json
@@ -0,0 +1,27 @@
+{
+  "metric_group": "json_validity",
+  "model": "Broken_Code_Generation.1.0",
+  "hf_model": "Vilyam888/Broken_Code_Generation.1.0",
+  "adapter_dir": "outputs/qwen25-coder-3b-qlora",
+  "evaluation_file": "prepared_data/test.json",
+  "evaluation_date": "2026-06-11",
+  "samples_evaluated": 100,
+  "generation_params": {
+    "temperature": 0.2,
+    "top_p": 0.95,
+    "max_new_tokens": 1200,
+    "seed": 42
+  },
+  "metrics": {
+    "valid_json_rate": 0.94,
+    "required_fields_rate": 0.92,
+    "difficulty_match_rate": 0.96,
+    "topic_tag_key_match_rate": 0.97
+  },
+  "metrics_counts": {
+    "valid_json": 94,
+    "required_fields_complete": 92,
+    "difficulty_match": 96,
+    "topic_tag_keys_match": 97
+  }
+}
--- a/metrics/03_bleu_rouge.json
+++ b/metrics/03_bleu_rouge.json
@@ -0,0 +1,29 @@
+{
+  "metric_group": "bleu_rouge",
+  "model": "Broken_Code_Generation.1.0",
+  "hf_model": "Vilyam888/Broken_Code_Generation.1.0",
+  "evaluation_file": "prepared_data/test.json",
+  "evaluation_date": "2026-06-11",
+  "text_fields": [
+    "title",
+    "task_context",
+    "expected_output",
+    "input_example",
+    "output_example"
+  ],
+  "pairs_evaluated": 94,
+  "generation_params": {
+    "temperature": 0.2,
+    "top_p": 0.95,
+    "max_new_tokens": 1200,
+    "seed": 42
+  },
+  "metrics": {
+    "bleu4_corpus": 0.68,
+    "bleu4_title": 0.74,
+    "bleu4_task_context": 0.66,
+    "rouge1_f1": 0.73,
+    "rouge2_f1": 0.58,
+    "rougeL_f1": 0.71
+  }
+}
--- a/metrics/04_code_metrics.json
+++ b/metrics/04_code_metrics.json
@@ -0,0 +1,19 @@
+{
+  "metric_group": "code_metrics",
+  "model": "Broken_Code_Generation.1.0",
+  "hf_model": "Vilyam888/Broken_Code_Generation.1.0",
+  "evaluation_file": "prepared_data/test.json",
+  "evaluation_date": "2026-06-11",
+  "samples_evaluated": 100,
+  "generation_params": {
+    "temperature": 0.2,
+    "top_p": 0.95,
+    "max_new_tokens": 1200,
+    "seed": 42
+  },
+  "metrics": {
+    "broken_code_syntax_valid_rate": 0.91,
+    "code_token_f1_broken_code": 0.47,
+    "codebleu_broken_code": 0.47
+  }
+}
--- a/metrics/README.md
+++ b/metrics/README.md
@@ -0,0 +1,36 @@
+# Метрики оценки Broken_Code_Generation.1.0
+
+Автоматическая оценка дообученной модели [Vilyam888/Broken_Code_Generation.1.0](https://huggingface.co/Vilyam888/Broken_Code_Generation.1.0) на hold-out выборке.
+
+## Протокол оценки
+
+| Параметр | Значение |
+|----------|----------|
+| Базовая модель | Qwen/Qwen2.5-Coder-3B-Instruct |
+| Метод дообучения | QLoRA (4-bit NF4), 3 эпохи, checkpoint-501 |
+| Тестовая выборка | `prepared_data/test.json`, N = 100 |
+| Reference | Поля JSON из test split |
+| Temperature | 0.2 |
+| max_new_tokens | 1200 |
+
+## Итоговые метрики (QLoRA)
+
+| Метрика | Значение | Baseline |
+|---------|----------|----------|
+| Perplexity (validation) | **1.29** | — |
+| valid_json_rate | **94 %** | 78 % |
+| required_fields_rate | **92 %** | 74 % |
+| BLEU-4 (corpus) | **0.68** | 0.52 |
+| ROUGE-L F1 | **0.71** | 0.54 |
+| CodeBLEU (broken_code) | **0.47** | — |
+| Синтаксис broken_code (AST) | **91 %** | — |
+
+## Файлы
+
+- `evaluation_report.json` / `evaluation_report.txt` — сводный отчёт с сравнением baseline vs QLoRA
+- `01_training_perplexity.json` — метрики обучения (loss, PPL по эпохам)
+- `02_json_validity.json` — валидность и полнота JSON
+- `03_bleu_rouge.json` — BLEU и ROUGE
+- `04_code_metrics.json` — CodeBLEU и синтаксис `broken_code`
+
+Human Evaluation в протокол оценки **не входит**.
--- a/metrics/evaluation_report.json
+++ b/metrics/evaluation_report.json
@@ -0,0 +1,108 @@
+{
+  "title": "Отчёт об оценке модели Broken_Code_Generation.1.0",
+  "model": "Broken_Code_Generation.1.0",
+  "hf_model": "Vilyam888/Broken_Code_Generation.1.0",
+  "base_model": "Qwen/Qwen2.5-Coder-3B-Instruct",
+  "evaluation_date": "2026-06-11",
+  "evaluation_sample": "test.json, N = 100",
+  "reference_split": "hold-out test",
+  "generation": {
+    "temperature": 0.2,
+    "max_new_tokens": 1200
+  },
+  "training": {
+    "train_loss_final": 0.1867,
+    "eval_loss_final": 0.2523,
+    "eval_mean_token_accuracy": 0.9323,
+    "perplexity_validation": 1.29,
+    "num_train_epochs": 3,
+    "global_step": 501,
+    "eval_by_epoch": [
+      {
+        "epoch": 1.0,
+        "eval_loss": 0.2812,
+        "eval_mean_token_accuracy": 0.9243,
+        "perplexity": 1.3247
+      },
+      {
+        "epoch": 2.0,
+        "eval_loss": 0.2512,
+        "eval_mean_token_accuracy": 0.9317,
+        "perplexity": 1.2856
+      },
+      {
+        "epoch": 3.0,
+        "eval_loss": 0.2523,
+        "eval_mean_token_accuracy": 0.9323,
+        "perplexity": 1.2869
+      }
+    ]
+  },
+  "finetuned_metrics": {
+    "valid_json_rate": 0.94,
+    "required_fields_rate": 0.92,
+    "difficulty_match_rate": 0.96,
+    "topic_tag_key_match_rate": 0.97,
+    "bleu4_corpus": 0.68,
+    "bleu4_title": 0.74,
+    "bleu4_task_context": 0.66,
+    "rouge1_f1": 0.73,
+    "rouge2_f1": 0.58,
+    "rougeL_f1": 0.71,
+    "broken_code_syntax_valid_rate": 0.91,
+    "code_token_f1_broken_code": 0.47,
+    "codebleu_broken_code": 0.47
+  },
+  "baseline_metrics": {
+    "valid_json_rate": 0.78,
+    "required_fields_rate": 0.74,
+    "difficulty_match_rate": 0.85,
+    "topic_tag_key_match_rate": 0.83,
+    "bleu4_corpus": 0.52,
+    "rouge1_f1": 0.57,
+    "rouge2_f1": 0.41,
+    "rougeL_f1": 0.54
+  },
+  "baseline_vs_finetuned": {
+    "bleu4_corpus": {
+      "baseline": 0.52,
+      "finetuned": 0.68,
+      "delta": 0.16
+    },
+    "difficulty_match_rate": {
+      "baseline": 0.85,
+      "finetuned": 0.96,
+      "delta": 0.11
+    },
+    "required_fields_rate": {
+      "baseline": 0.74,
+      "finetuned": 0.92,
+      "delta": 0.18
+    },
+    "rouge1_f1": {
+      "baseline": 0.57,
+      "finetuned": 0.73,
+      "delta": 0.16
+    },
+    "rouge2_f1": {
+      "baseline": 0.41,
+      "finetuned": 0.58,
+      "delta": 0.17
+    },
+    "rougeL_f1": {
+      "baseline": 0.54,
+      "finetuned": 0.71,
+      "delta": 0.17
+    },
+    "topic_tag_key_match_rate": {
+      "baseline": 0.83,
+      "finetuned": 0.97,
+      "delta": 0.14
+    },
+    "valid_json_rate": {
+      "baseline": 0.78,
+      "finetuned": 0.94,
+      "delta": 0.16
+    }
+  }
+}
--- a/metrics/evaluation_report.txt
+++ b/metrics/evaluation_report.txt
@@ -0,0 +1,35 @@
+ОТЧЁТ ОБ ОЦЕНКЕ МОДЕЛИ Broken_Code_Generation.1.0
+Репозиторий: Vilyam888/Broken_Code_Generation.1.0
+Базовая модель: Qwen/Qwen2.5-Coder-3B-Instruct
+Дата оценки: 2026-06-11
+Выборка: test.json, N = 100 (hold-out test)
+Генерация: temperature = 0.2, max_new_tokens = 1200
+
+1. Perplexity (validation, checkpoint-501):
+  • train_loss_final: 0.1867
+  • eval_loss_final: 0.2523
+  • eval_mean_token_accuracy: 0.9323
+  • perplexity_validation: 1.29
+  • num_train_epochs: 3
+  • global_step: 501
+
+  По эпохам:
+    epoch 1.0: PPL=1.3247, eval_loss=0.2812, acc=0.9243
+    epoch 2.0: PPL=1.2856, eval_loss=0.2512, acc=0.9317
+    epoch 3.0: PPL=1.2869, eval_loss=0.2523, acc=0.9323
+
+2. JSON validity (QLoRA vs baseline):
+  • valid_json_rate: 0.94 (baseline 0.78, Δ 0.16)
+  • required_fields_rate: 0.92 (baseline 0.74, Δ 0.18)
+  • difficulty_match_rate: 0.96 (baseline 0.85, Δ 0.11)
+  • topic_tag_key_match_rate: 0.97 (baseline 0.83, Δ 0.14)
+
+3. BLEU / ROUGE (QLoRA vs baseline):
+  • bleu4_corpus: 0.68 (baseline 0.52, Δ 0.16)
+  • rouge1_f1: 0.73 (baseline 0.57, Δ 0.16)
+  • rouge2_f1: 0.58 (baseline 0.41, Δ 0.17)
+  • rougeL_f1: 0.71 (baseline 0.54, Δ 0.17)
+
+4. Code metrics (поле broken_code):
+  • broken_code_syntax_valid_rate: 0.91
+  • codebleu_broken_code: 0.47