From 8d914512abd3dcbdbb946bc1e625281b341db036 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sun, 12 Jul 2026 02:36:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Nextorage/Llama-3.1-Swallow-8B-OpenMath-FT Source: Original Platform --- .gitattributes | 36 + README.md | 160 +++ config.json | 96 ++ eval_utils.py | 272 +++++ model-00001.safetensors | 3 + model-00002.safetensors | 3 + model-00003.safetensors | 3 + model-00004.safetensors | 3 + model.safetensors.index.json | 298 +++++ run_inference.py | 818 ++++++++++++++ special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++++++++++++++++++++++++++++++++ 13 files changed, 3782 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 eval_utils.py create mode 100644 model-00001.safetensors create mode 100644 model-00002.safetensors create mode 100644 model-00003.safetensors create mode 100644 model-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 run_inference.py create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..9ca45a7 --- /dev/null +++ b/README.md @@ -0,0 +1,160 @@ +--- +language: + - ja +license: llama3 +base_model: tokyotech-llm/Llama-3.1-Swallow-8B +library_name: transformers +tags: + - fine-tuned + - japanese + - math + - openmath + - code-generation + - text-generation +datasets: + - nvidia/OpenMathInstruct-1 +pipeline_tag: text-generation +--- + +# Llama-3.1-Swallow-8B OpenMath Fine-Tuned (T2T) + +## 概要 / Overview + +Nextorage **AiDAPTIV+** プラットフォーム上で [Llama-3.1-Swallow-8B](https://huggingface.co/tokyotech-llm/Llama-3.1-Swallow-8B) をフルファインチューニングした、日本語数学文章題解答モデルです。 + +Python コード生成 + コード実行パイプラインと組み合わせることで Exact Match **60.0%** を達成し、商用 API（Claude Sonnet・GPT-4o: 各 56.7%）を上回りました。 + +A full fine-tuned version of [Llama-3.1-Swallow-8B](https://huggingface.co/tokyotech-llm/Llama-3.1-Swallow-8B) for Japanese math word problem solving, trained on AiDAPTIV+ platform by Nextorage. +Combined with a Python code-execution pipeline, it achieves Exact Match **60.0%** — surpassing Claude Sonnet and GPT-4o (both 56.7%) on the same test set. + +--- + +## 性能 / Performance + +テストセット: OpenMath Instruct 日本語版 30件（コード実行パイプライン使用） + +| モデル | Exact Match | +|--------|:-----------:| +| Llama-3.1-Swallow-8B（未FT ベースライン） | 36.7% | +| Claude Sonnet (format_compliant) | 56.7% | +| GPT-4o (format_compliant) | 56.7% | +| **本モデル（Full FT + コード実行パイプライン）** | **60.0%** | + +> **注**: コード実行パイプラインなしでは 20.0%。パイプラインにより 3 倍の精度向上。 + +--- + +## 使い方 / Usage + +### 推論スクリプト（コード実行パイプライン付き） + +```bash +# 依存パッケージのインストール +pip install transformers torch + +# 評価・推論の実行（コード実行パイプライン有効） +python run_inference.py \ + --model_path /path/to/this/model \ + --test_data /path/to/test.json \ + --code_exec_pipeline \ + --output_dir ./results +``` + +### Python での直接推論 + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +model_path = "Nextorage/Llama-3.1-Swallow-8B-OpenMath-FT" +tokenizer = AutoTokenizer.from_pretrained(model_path) +model = AutoModelForCausalLM.from_pretrained( + model_path, + torch_dtype=torch.bfloat16, + device_map="auto" +) + +SYSTEM_PROMPT = """あなたは数学の問題を解く優秀なAIアシスタントです。ステップバイステップで考え、Pythonコードを使って計算し、最終的な答えを明示してください。 + +回答は以下のフォーマットに従ってください： +1. Pythonコードは と タグで囲んでください +2. コードの実行結果は と タグで囲んでください +3. 最終的な答えは \\boxed{答え} の形式で明示してください""" + +question = "ジェイデンは8台のおもちゃの車を持っています。3台を友人に譲りました。ジェイデンには何台のおもちゃの車が残っていますか？" + +messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": question}, +] +input_ids = tokenizer.apply_chat_template(messages, return_tensors="pt").to(model.device) +output = model.generate(input_ids, max_new_tokens=512) +print(tokenizer.decode(output[0][input_ids.shape[1]:], skip_special_tokens=True)) +``` + +**期待出力例:** +``` +Pythonコードを使用してこの問題を解決しましょう。 +initial_cars = 8 +given_away = 3 +remaining = initial_cars - given_away +print(remaining) + +5 + +したがって、ジェイデンには \boxed{5} 台のおもちゃの車が残っています。 +``` + +--- + +## 学習設定 / Training Configuration + +| パラメータ | 値 | +|-----------|-----| +| ベースモデル | tokyotech-llm/Llama-3.1-Swallow-8B | +| 手法 | Full Fine-Tuning | +| 学習データ | OpenMath Instruct 日本語版（9,772件 クリーニング済み） | +| Learning Rate | 1e-6 | +| LR Scheduler | cosine | +| Epoch | 1 | +| Batch Size (effective) | 32 (per_device=2, grad_accum=16) | +| Max Seq Length | 2048 | +| Precision | bf16_mixed | +| Weight Decay | 0.05 | +| プラットフォーム | Nextorage AiDAPTIV+ (phisonai2) | + +### データセット詳細 + +- 元データ: [NVIDIA OpenMathInstruct-1](https://huggingface.co/datasets/nvidia/OpenMathInstruct-1) 日本語翻訳版（全 1,825,008 件） +- `` タグ付きエントリを抽出後、Python 実行検証でクリーニング（除外率 2.3%） +- GSM8K・MATH 等の数学ベンチマークを元に構築された数学文章題 + Python コード解答ペア + +--- + +## 制限事項 / Limitations + +- 日本語数学文章題（主に GSM8K・MATH 難易度）に特化しており、他タスクの性能は保証されない +- 最高精度の発揮には Python コード実行パイプライン（`subprocess` 実行環境）が必要 +- 日本語翻訳は機械翻訳を使用しており、一部の表現に不自然さが残る場合がある +- テストセット 30件での評価結果のため、統計的信頼区間に注意 + +--- + +## ライセンス / License + +本モデルは [Llama 3.1 Community License](https://llama.meta.com/llama3_1/license/) に基づいています。 +商用利用は条件付きで許可されています。詳細はライセンス全文を参照してください。 + +--- + +## 引用 / Citation + +```bibtex +@misc{nextorage-openmath-ft-2026, + title = {Llama-3.1-Swallow-8B OpenMath Fine-Tuned}, + author = {Nextorage Inc.}, + year = {2026}, + howpublished = {\url{https://huggingface.co/Nextorage/Llama-3.1-Swallow-8B-OpenMath-FT}}, + note = {Full fine-tuned model for Japanese math word problem solving on AiDAPTIV+ platform} +} +``` diff --git a/config.json b/config.json new file mode 100644 index 0000000..02b1ecb --- /dev/null +++ b/config.json @@ -0,0 +1,96 @@ +{ + "vocab_size": 128256, + "max_position_embeddings": 8192, + "hidden_size": 4096, + "intermediate_size": 14336, + "num_hidden_layers": 32, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "hidden_act": "silu", + "initializer_range": 0.02, + "rms_norm_eps": 1e-05, + "pretraining_tp": 1, + "use_cache": true, + "rope_theta": 500000.0, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "attention_bias": false, + "attention_dropout": 0.0, + "mlp_bias": false, + "head_dim": 128, + "return_dict": true, + "output_hidden_states": false, + "output_attentions": false, + "torchscript": false, + "torch_dtype": "bfloat16", + "use_bfloat16": false, + "tf_legacy_loss": false, + "pruned_heads": {}, + "tie_word_embeddings": false, + "chunk_size_feed_forward": 0, + "is_encoder_decoder": false, + "is_decoder": false, + "cross_attention_hidden_size": null, + "add_cross_attention": false, + "tie_encoder_decoder": false, + "max_length": 20, + "min_length": 0, + "do_sample": false, + "early_stopping": false, + "num_beams": 1, + "num_beam_groups": 1, + "diversity_penalty": 0.0, + "temperature": 1.0, + "top_k": 50, + "top_p": 1.0, + "typical_p": 1.0, + "repetition_penalty": 1.0, + "length_penalty": 1.0, + "no_repeat_ngram_size": 0, + "encoder_no_repeat_ngram_size": 0, + "bad_words_ids": null, + "num_return_sequences": 1, + "output_scores": false, + "return_dict_in_generate": false, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "remove_invalid_values": false, + "exponential_decay_length_penalty": null, + "suppress_tokens": null, + "begin_suppress_tokens": null, + "architectures": [ + "LlamaForCausalLM" + ], + "finetuning_task": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "tokenizer_class": null, + "prefix": null, + "bos_token_id": 128000, + "pad_token_id": null, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "sep_token_id": null, + "decoder_start_token_id": null, + "task_specific_params": null, + "problem_type": null, + "_name_or_path": "/home/mshohda/Desktop/KambayashiWork/Nextorage-LLM-ChatBot/AiDaptive_Software/models/Llama-3.1-Swallow-8B", + "_attn_implementation_autoset": false, + "transformers_version": "4.51.3", + "model_type": "llama", + "use_flash_attention": true +} diff --git a/eval_utils.py b/eval_utils.py new file mode 100644 index 0000000..98f2001 --- /dev/null +++ b/eval_utils.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +""" +評価ユーティリティ: boxed answer抽出、数学的等価判定、コード実行、フォーマットチェック +""" + +import re +import subprocess +import tempfile +import math +from typing import Optional + + +# ============================================================================= +# Boxed Answer Extraction +# ============================================================================= + +def extract_boxed_answer(text: str) -> Optional[str]: + """ + \\boxed{...} から最終回答を抽出する。 + ネスト対応: \\boxed{\\frac{1}{2}} のような場合も正しく抽出。 + 複数ある場合は最後のものを返す (最終回答は通常末尾)。 + 全角括弧（）や日本語「箱に入れ」表記にも対応。 + """ + # 表記ゆれに対応: \boxed, \\boxed, \Boxed, \\Boxed + 全角括弧 + pattern = r'\\?\\?[Bb]oxed\s*[\{（]' + matches = list(re.finditer(pattern, text)) + + # フォールバック: 日本語「箱に入れ」パターン + if not matches: + jp_pattern = r'[\{（]([^}）]+)[}）]\s*(?:を箱に入れ|を.*?箱)' + jp_match = re.search(jp_pattern, text) + if jp_match: + return jp_match.group(1).strip() + return None + + # 最後のマッチから抽出 (最終回答) + last_match = matches[-1] + start = last_match.end() # '{' または '（' の直後 + + # ネスト対応のブレースマッチング（全角括弧対応） + depth = 1 + i = start + while i < len(text) and depth > 0: + if text[i] in ('{', '（'): + depth += 1 + elif text[i] in ('}', '）'): + depth -= 1 + i += 1 + + if depth == 0: + return text[start:i - 1].strip() + return None + + +def normalize_answer(answer: str) -> str: + """回答文字列を正規化する (比較用)""" + if answer is None: + return "" + s = answer.strip() + # $記号を除去 + s = s.replace("$", "") + # LaTeXコマンドの正規化 + s = s.replace("\\%", "%") + s = s.replace("\\$", "$") + # LaTeXコマンド周辺のスペースを正規化 + # "\\ frac {a} {b}" → "\frac{a}{b}" + s = re.sub(r'\\+\s*frac\s*', r'\\frac', s) + s = re.sub(r'(\\frac)\s*\{', r'\1{', s) + s = re.sub(r'\}\s*\{', '}{', s) + # 余分な空白を除去 + s = re.sub(r'\s+', ' ', s).strip() + return s + + +def math_equivalent(pred: str, gold: str) -> bool: + """ + 2つの数学的表現が等価かどうかを判定する。 + sympyを使った数式パースを試み、失敗した場合は文字列比較にフォールバック。 + """ + pred_norm = normalize_answer(pred) + gold_norm = normalize_answer(gold) + + if not pred_norm or not gold_norm: + return False + + # 1. 完全一致 + if pred_norm == gold_norm: + return True + + # 2. 数値比較 (小数・整数) + try: + pred_val = float(eval_simple_expr(pred_norm)) + gold_val = float(eval_simple_expr(gold_norm)) + if math.isclose(pred_val, gold_val, rel_tol=1e-6, abs_tol=1e-9): + return True + except (ValueError, TypeError, SyntaxError, ZeroDivisionError): + pass + + # 3. sympy による数式等価判定 + try: + import sympy + from sympy.parsing.latex import parse_latex + + # LaTeX表記をsympyで解析 + try: + pred_expr = parse_latex(pred_norm) + gold_expr = parse_latex(gold_norm) + except Exception: + # LaTeXパースが失敗した場合、sympy.sympifyを試す + pred_expr = sympy.sympify(pred_norm) + gold_expr = sympy.sympify(gold_norm) + + # 数値的に等しいか + diff = sympy.simplify(pred_expr - gold_expr) + if diff == 0: + return True + + # 数値に変換して比較 + pred_float = float(pred_expr.evalf()) + gold_float = float(gold_expr.evalf()) + if math.isclose(pred_float, gold_float, rel_tol=1e-6, abs_tol=1e-9): + return True + except Exception: + pass + + # 4. 文字列の緩い比較 (空白、カンマ区切り等を無視) + pred_clean = re.sub(r'[,\s\\{}]', '', pred_norm.lower()) + gold_clean = re.sub(r'[,\s\\{}]', '', gold_norm.lower()) + if pred_clean == gold_clean: + return True + + return False + + +def eval_simple_expr(s: str) -> float: + """簡単な数式文字列を評価する (分数、パーセント等に対応)""" + s = s.strip() + # パーセント + if s.endswith('%'): + return float(s[:-1]) + # LaTeX分数 \frac{a}{b} + frac_match = re.match(r'\\frac\s*\{([^}]+)\}\s*\{([^}]+)\}', s) + if frac_match: + num = float(frac_match.group(1)) + den = float(frac_match.group(2)) + return num / den + # 通常の分数 a/b + if '/' in s and not any(c.isalpha() for c in s): + parts = s.split('/') + if len(parts) == 2: + return float(parts[0]) / float(parts[1]) + return float(s) + + +# ============================================================================= +# Code Execution +# ============================================================================= + +def extract_code_blocks(text: str) -> list[str]: + """... タグからPythonコードブロックを抽出""" + pattern = r'(.*?)' + matches = re.findall(pattern, text, re.DOTALL) + return [m.strip() for m in matches if m.strip()] + + +def _auto_print_last_expr(code: str) -> str: + """ + コードの最終行が print() を含まない式の場合、自動で print() を付与する。 + Jupyter ノートブック形式のコード（最終行が式評価のみ）に対応。 + """ + lines = code.rstrip().split("\n") + if not lines: + return code + + last_line = lines[-1].strip() + + # 空行、コメント、代入文、制御文、print文はスキップ + if (not last_line + or last_line.startswith("#") + or "=" in last_line and not last_line.startswith("=") and "==" not in last_line + or last_line.startswith(("if ", "for ", "while ", "def ", "class ", "import ", + "from ", "return ", "try:", "except", "with ", "raise ")) + or "print(" in last_line): + return code + + # 最終行が式→ print() で囲む + lines[-1] = f"print({last_line})" + return "\n".join(lines) + + +def execute_code_safely(code: str, timeout: int = 10) -> dict: + """ + Pythonコードをサブプロセスで安全に実行する。 + 最終行が式の場合は自動で print() を付与する。 + Returns: {"success": bool, "stdout": str, "stderr": str} + """ + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + # sympy は頻出なのでimportを追加 + code_with_print = _auto_print_last_expr(code) + wrapped = "import sympy\nfrom sympy import *\n" + code_with_print + f.write(wrapped) + f.flush() + try: + result = subprocess.run( + ['python3', f.name], + capture_output=True, + text=True, + timeout=timeout, + ) + return { + "success": result.returncode == 0, + "stdout": result.stdout.strip(), + "stderr": result.stderr.strip(), + } + except subprocess.TimeoutExpired: + return { + "success": False, + "stdout": "", + "stderr": f"Timeout after {timeout}s", + } + except Exception as e: + return { + "success": False, + "stdout": "", + "stderr": str(e), + } + + +def check_code_execution(text: str) -> dict: + """ + テキスト中のコードブロックを実行し、結果をまとめる。 + Returns: {"has_code": bool, "num_blocks": int, "all_success": bool, "results": list} + """ + blocks = extract_code_blocks(text) + if not blocks: + return {"has_code": False, "num_blocks": 0, "all_success": True, "results": []} + + results = [] + for code in blocks: + result = execute_code_safely(code) + results.append(result) + + return { + "has_code": True, + "num_blocks": len(blocks), + "all_success": all(r["success"] for r in results), + "results": results, + } + + +# ============================================================================= +# Format Compliance +# ============================================================================= + +def check_format_compliance(text: str) -> dict: + """回答のフォーマット遵守率をチェック""" + has_boxed = bool(re.search(r'\\?\\?[Bb]oxed\s*\{', text)) + + # コードタグの整合性 + code_opens = len(re.findall(r'', text)) + code_closes = len(re.findall(r'', text)) + code_tags_balanced = code_opens == code_closes + + # 不正なトークンが含まれていないか + has_bad_tokens = bool(re.search(r'<\|im_end\|>|<\|im_start\|>|<\|endoftext\|>', text)) + + return { + "has_boxed_answer": has_boxed, + "code_tags_balanced": code_tags_balanced, + "no_bad_tokens": not has_bad_tokens, + "format_ok": has_boxed and code_tags_balanced and not has_bad_tokens, + } diff --git a/model-00001.safetensors b/model-00001.safetensors new file mode 100644 index 0000000..499e294 --- /dev/null +++ b/model-00001.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b1fd077c84463ecae94bd1bae98458f94696447e16fc8fe90260b8ddf28f082 +size 5295466472 diff --git a/model-00002.safetensors b/model-00002.safetensors new file mode 100644 index 0000000..dbd79bb --- /dev/null +++ b/model-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ecc1f36f74badd76cbb7fb14e1cdf4c05b3eac307213d16e3f3a7394ad0c187 +size 5352157840 diff --git a/model-00003.safetensors b/model-00003.safetensors new file mode 100644 index 0000000..fdf1f51 --- /dev/null +++ b/model-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75ca19d05917ba3e2a051fb7145dbdeffb9fe4e050f4c68f51ad345b3045fafa +size 4362258776 diff --git a/model-00004.safetensors b/model-00004.safetensors new file mode 100644 index 0000000..889b559 --- /dev/null +++ b/model-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:178d1d57cfaca0e351b6dd0ab8fb39dcfd60b73be9800a8f5cf8dc4828c0cb19 +size 1050673280 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..29c940c --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 16060522496 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003.safetensors", + "model.norm.weight": "model-00003.safetensors", + "lm_head.weight": "model-00004.safetensors" + } +} \ No newline at end of file diff --git a/run_inference.py b/run_inference.py new file mode 100644 index 0000000..b0ae3e9 --- /dev/null +++ b/run_inference.py @@ -0,0 +1,818 @@ +#!/usr/bin/env python3 +""" +OpenMath Finetuning 自動評価スクリプト (Llama-3.1-Swallow-8B) + +finetuned model でテストデータに対して推論を実行し、 +自動採点 (Exact Match, Code Execution, Format Compliance) を全件実行。 +結果をJSON + CSV + サマリーレポートで保存。 + +Usage: + python3 run_evaluation.py + python3 run_evaluation.py --model_path /path/to/model --test_data /path/to/test.json + python3 run_evaluation.py --skip_inference --results_json /path/to/existing_results.json +""" + +import os +os.environ["CUDA_LAUNCH_BLOCKING"] = "1" + +import argparse +import re +import csv +import gc +import json +import time +from datetime import datetime +from pathlib import Path + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList +try: + from peft import PeftModel + _PEFT_AVAILABLE = True +except ImportError: + _PEFT_AVAILABLE = False + +from eval_utils import ( + extract_boxed_answer, + math_equivalent, + check_code_execution, + check_format_compliance, + extract_code_blocks, + execute_code_safely, +) + +# デフォルトパス (Llama-3.1-Swallow-8B) +DEFAULT_FINETUNED_MODEL = "/home/mshohda/Desktop/KambayashiWork/Nextorage-LLM-ChatBot/AiDaptive_Software/output_llama_openmath" +DEFAULT_ORIGINAL_MODEL = "/home/mshohda/Desktop/KambayashiWork/Nextorage-LLM-ChatBot/AiDaptive_Software/models/Llama-3.1-Swallow-8B" +DEFAULT_TEST_DATA = "/home/mshohda/Desktop/KambayashiWork/Nextorage-LLM-ChatBot/Dataset/FinetuiningDataset/OpenMathInstruct/openmath_test_500.json" +DEFAULT_OUTPUT_DIR = "/home/mshohda/Desktop/KambayashiWork/Nextorage-LLM-ChatBot/AiDaptive_Software/vNXUN_2_03_00/commands/finetuning_openmath_llama/evaluation/results" + +SYSTEM_PROMPT = ( + "あなたは数学の問題を解く優秀なAIアシスタントです。ステップバイステップで考え、Pythonコードを使って計算し、最終的な答えを明示してください。\n" + "\n" + "回答は以下のフォーマットに従ってください：\n" + "1. Pythonコードは と タグで囲んでください\n" + "2. コードの実行結果は と タグで囲んでください\n" + "3. 最終的な答えは \\boxed{答え} の形式で明示してください\n" + "\n" + "回答例：\n" + "Pythonコードを使用してこの問題を解決しましょう。\n" + "x = 2 + 3\n" + "print(x)\n" + "\n" + "5\n" + "\n" + "したがって、答えは\\boxed{5}です。" +) + + +def parse_args(): + parser = argparse.ArgumentParser(description="OpenMath Finetuning Evaluation (Llama)") + parser.add_argument("--model_path", default=DEFAULT_FINETUNED_MODEL, + help="Path to finetuned model") + parser.add_argument("--original_model_path", default=DEFAULT_ORIGINAL_MODEL, + help="Path to original model (for tokenizer)") + parser.add_argument("--test_data", default=DEFAULT_TEST_DATA, + help="Path to test dataset JSON") + parser.add_argument("--output_dir", default=DEFAULT_OUTPUT_DIR, + help="Directory to save results") + parser.add_argument("--max_samples", type=int, default=-1, + help="Max samples to evaluate (-1 for all)") + parser.add_argument("--max_new_tokens", type=int, default=1024, + help="Max new tokens for generation") + parser.add_argument("--gpu_id", type=int, default=0, + help="GPU ID to use") + parser.add_argument("--skip_inference", action="store_true", + help="Skip inference, only run scoring on existing results") + parser.add_argument("--results_json", default=None, + help="Path to existing results JSON (for --skip_inference)") + parser.add_argument("--skip_code_exec", action="store_true", + help="Skip code execution checks") + parser.add_argument("--model_label", default=None, + help="Label for this model (e.g. 'finetuned', 'original'). " + "Used in output filenames and reports.") + parser.add_argument("--system_prompt", default=None, + help="Custom system prompt (overrides default SYSTEM_PROMPT). " + "Use to provide format instructions for baseline models.") + parser.add_argument("--code_exec_pipeline", action="store_true", + help="Enable code execution pipeline: execute blocks " + "and inject real outputs into ") + parser.add_argument("--max_retries", type=int, default=2, + help="Max self-repair retries on code execution error (default: 2)") + return parser.parse_args() + + +def build_prompt(question: str) -> str: + """学習時と同じQAフォーマットでプロンプトを構築 (Llama 3.1形式)""" + return ( + f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n" + f"{SYSTEM_PROMPT}<|eot_id|>" + f"<|start_header_id|>user<|end_header_id|>\n\n" + f"{question}<|eot_id|>" + f"<|start_header_id|>assistant<|end_header_id|>\n\n" + ) + + +def build_repair_prompt(error_msg: str, failed_code: str, attempt: int) -> str: + """エラー箇所を特定し、具体的な修正指示を生成""" + # tracebackからエラー行を抽出 + error_line = "" + line_match = re.search(r'File ".*", line (\d+)', error_msg) + if line_match: + line_no = int(line_match.group(1)) + code_lines = failed_code.strip().split('\n') + adjusted = line_no - 2 # wrapper scriptのオフセット補正 + if 0 <= adjusted < len(code_lines): + error_line = code_lines[adjusted].strip() + + # 2回目以降はアプローチ転換を促す + approach_change = "" + if attempt >= 1: + approach_change = ( + "前回と同じアプローチでは解決しません。" + "sympyの代わりにmathモジュールやforループなど、別の方法を試してください。" + ) + + repair = f"\nError: {error_msg}\n\n" + repair += "上記のコードでエラーが発生しました。" + if error_line: + repair += f"特にこの行が問題です: `{error_line}`\n" + repair += "エラーメッセージをよく読み、問題の箇所を変更して修正したコードを書いてください。" + if approach_change: + repair += f"\n{approach_change}" + repair += "\n" + + return repair + + +def postprocess_response(text: str) -> str: + """回答後の繰り返し生成を除去する。""" + # 1. ターンマーカー検出 + match = re.search(r'\n[^\n]{0,30}?(?:user|assistant)', text) + if match: + text = text[:match.start()].strip() + + # 2. 短フレーズの繰り返し検出 (3回以上同じフレーズが繰り返されたら切り取る) + match = re.search(r'(.{10,80}?)\1{2,}', text, re.DOTALL) + if match: + text = text[:match.start() + len(match.group(1))].strip() + + return text.strip() + + +def _find_lora_adapter(model_path: str) -> str | None: + """ + phisonai2 LoRA学習時のアダプタパスを探す。 + model_path: .../output_dir/finetuned_model_YYYY-mm-dd-HH-MM-SS/epoch_N_step_M_#ModelName + 対応するLoRAアダプタ: .../lora_adapters/finetuned_model_.../Lora_epoch_N_step_M_#ModelName + """ + model_path = Path(model_path) + # epoch_N_step_M_#ModelName → Lora_epoch_N_step_M_#ModelName + step_dir_name = model_path.name + lora_step_name = "Lora_" + step_dir_name + + # output_dir/lora_adapters/finetuned_model_.../Lora_epoch_N_step_M_#... + run_dir = model_path.parent # finetuned_model_YYYY-... + output_dir = run_dir.parent # output_dir (e.g. /mnt/nvme0/output_llama_lora/lora_01_...) + candidate = output_dir / "lora_adapters" / run_dir.name / lora_step_name + if (candidate / "adapter_config.json").exists(): + return str(candidate) + return None + + +def load_model(model_path: str, tokenizer_path: str, gpu_id: int): + """モデルとトークナイザーをロード。LoRAアダプタが存在する場合は自動適用。""" + # CUDA_VISIBLE_DEVICES でGPUを制限 + os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id) + logical_gpu = 0 # CUDA_VISIBLE_DEVICES で絞った後は論理GPU 0 + + print(f"Loading tokenizer from: {tokenizer_path}") + tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + # LoRAアダプタの存在確認 + lora_adapter_path = _find_lora_adapter(model_path) + if lora_adapter_path: + if not _PEFT_AVAILABLE: + raise ImportError("peft is required for LoRA evaluation. Run: pip install peft") + print(f"LoRA adapter found: {lora_adapter_path}") + print(f"Loading base model from: {tokenizer_path}") + model = AutoModelForCausalLM.from_pretrained( + tokenizer_path, + torch_dtype=torch.bfloat16, + device_map={"": logical_gpu}, + trust_remote_code=True, + ) + print(f"Applying LoRA adapter...") + model = PeftModel.from_pretrained(model, lora_adapter_path) + model = model.merge_and_unload() + print(f" [OK] LoRA merged into base model") + else: + print(f"Loading model from: {model_path}") + model = AutoModelForCausalLM.from_pretrained( + model_path, + torch_dtype=torch.bfloat16, + device_map={"": logical_gpu}, + trust_remote_code=True, + ) + + # vocab_size不一致の場合はリサイズ + actual_embed_size = model.model.embed_tokens.weight.shape[0] + tokenizer_vocab_size = len(tokenizer) + if actual_embed_size != tokenizer_vocab_size: + print(f" Resizing embeddings: {actual_embed_size} -> {tokenizer_vocab_size}") + model.resize_token_embeddings(tokenizer_vocab_size) + + model.config.pad_token_id = tokenizer.pad_token_id + model.eval() + print(f" [OK] Model loaded on GPU {gpu_id} (CUDA_VISIBLE_DEVICES={gpu_id})") + return model, tokenizer + + +def compute_perplexity(model, tokenizer, question: str, answer: str) -> float: + """質問と回答のペアに対するperplexityを計算""" + prompt = build_prompt(question) + answer + inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=2048).to(model.device) + + with torch.no_grad(): + outputs = model( + input_ids=inputs["input_ids"], + labels=inputs["input_ids"], + ) + + return torch.exp(outputs.loss).item() + + +def generate_response(model, tokenizer, question: str, max_new_tokens: int) -> str: + """プロンプトを生成してモデルから応答を取得""" + prompt = build_prompt(question) + inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + pad_token_id=tokenizer.pad_token_id, + eos_token_id=tokenizer.eos_token_id, + repetition_penalty=1.1, + ) + + response_ids = outputs[0][inputs["input_ids"].shape[1]:] + raw = tokenizer.decode(response_ids, skip_special_tokens=True) + return postprocess_response(raw) + + +class StopOnString(StoppingCriteria): + """指定文字列が生成テキストに出現したら生成を停止する""" + + def __init__(self, stop_string: str, tokenizer, prompt_length: int): + self.stop_string = stop_string + self.tokenizer = tokenizer + self.prompt_length = prompt_length + # stop_string のトークン数の3倍をチェック窓とする（分割トークン化対策） + self.check_window = max( + len(tokenizer.encode(stop_string, add_special_tokens=False)) * 3, 20 + ) + + def __call__(self, input_ids, scores, **kwargs): + if input_ids.shape[1] <= self.prompt_length: + return False + start = max(self.prompt_length, input_ids.shape[1] - self.check_window) + tail = self.tokenizer.decode(input_ids[0][start:], skip_special_tokens=True) + return self.stop_string in tail + + +def generate_with_code_exec( + model, tokenizer, question: str, max_new_tokens: int, max_retries: int = 2 +) -> tuple: + """ + コード実行パイプライン付き推論。 + + フロー: + 1. まで生成（コードブロックを含むテキスト） + 2. コードを実際にPythonで実行 + 3. 実行結果を に注入 + 4. 続きを生成して \\boxed{} を含む最終回答を取得 + 5. 実行エラー時はエラーを返してリトライ（self-repair） + + Returns: + (response_text, pipeline_info_dict) + """ + prompt = build_prompt(question) + input_ids = tokenizer(prompt, return_tensors="pt").input_ids.to(model.device) + prompt_len = input_ids.shape[1] + + pipeline_info = { + "code_exec_pipeline": True, + "attempts": [], + "total_attempts": 0, + "final_status": "no_code", + } + + # Phase 1: まで生成 + stop_criteria = StopOnString("", tokenizer, prompt_len) + + with torch.no_grad(): + outputs = model.generate( + input_ids=input_ids, + max_new_tokens=max_new_tokens, + do_sample=False, + pad_token_id=tokenizer.pad_token_id, + eos_token_id=tokenizer.eos_token_id, + repetition_penalty=1.1, + stopping_criteria=StoppingCriteriaList([stop_criteria]), + ) + + generated = tokenizer.decode(outputs[0][prompt_len:], skip_special_tokens=True) + + # コードブロックがない場合はそのまま返す + if "" not in generated: + pipeline_info["final_status"] = "no_code" + return postprocess_response(generated), pipeline_info + + # 以降を切り捨て（ハルシネーション防止） + code_end_idx = generated.index("") + len("") + response_so_far = generated[:code_end_idx] + + prev_code = None + for attempt in range(max_retries + 1): + pipeline_info["total_attempts"] = attempt + 1 + + # コードブロック抽出・実行 + code_blocks = extract_code_blocks(response_so_far) + if not code_blocks: + pipeline_info["final_status"] = "no_code_block" + break + + last_code = code_blocks[-1] + + # 同一コード検出（無限ループ防止） + if prev_code is not None and last_code.strip() == prev_code.strip(): + pipeline_info["final_status"] = "same_code_detected" + break + prev_code = last_code + + exec_result = execute_code_safely(last_code) + + attempt_info = { + "attempt": attempt, + "code_preview": last_code[:200], + "success": exec_result["success"], + "stdout": exec_result["stdout"][:500] if exec_result["stdout"] else "", + "stderr": exec_result["stderr"][:500] if exec_result["stderr"] else "", + } + pipeline_info["attempts"].append(attempt_info) + + if exec_result["success"]: + # 成功: 実行結果を注入して続きを生成 + output_text = exec_result["stdout"] if exec_result["stdout"] else "None" + injection = f"\n{output_text}\n\n" + response_so_far += injection + + # Phase 2: 最終回答を生成 + full_context = prompt + response_so_far + context_ids = tokenizer( + full_context, return_tensors="pt", truncation=True, max_length=4096 + ).input_ids.to(model.device) + context_len = context_ids.shape[1] + + with torch.no_grad(): + outputs = model.generate( + input_ids=context_ids, + max_new_tokens=max_new_tokens // 2, + do_sample=False, + pad_token_id=tokenizer.pad_token_id, + eos_token_id=tokenizer.eos_token_id, + repetition_penalty=1.1, + ) + + continuation = tokenizer.decode( + outputs[0][context_len:], skip_special_tokens=True + ) + response_so_far += continuation + pipeline_info["final_status"] = "success" + return postprocess_response(response_so_far), pipeline_info + + else: + # 実行失敗 + error_msg = exec_result["stderr"] or "Unknown error" + if len(error_msg) > 300: + error_msg = error_msg[:300] + "..." + + if attempt < max_retries: + # Self-repair: エラー箇所を特定した具体的な修正指示を生成 + repair_injection = build_repair_prompt(error_msg, last_code, attempt) + response_so_far += repair_injection + + full_context = prompt + response_so_far + context_ids = tokenizer( + full_context, return_tensors="pt", truncation=True, max_length=4096 + ).input_ids.to(model.device) + context_len = context_ids.shape[1] + + stop_criteria = StopOnString("", tokenizer, context_len) + + with torch.no_grad(): + outputs = model.generate( + input_ids=context_ids, + max_new_tokens=max_new_tokens // 2, + do_sample=False, + pad_token_id=tokenizer.pad_token_id, + eos_token_id=tokenizer.eos_token_id, + repetition_penalty=1.1, + stopping_criteria=StoppingCriteriaList([stop_criteria]), + ) + + repair_text = tokenizer.decode( + outputs[0][context_len:], skip_special_tokens=True + ) + + if "" in repair_text: + code_end = repair_text.index("") + len("") + response_so_far += repair_text[:code_end] + # 次のループで再実行（同一コード検出はループ先頭で行う） + else: + # モデルがコードブロックを生成しなかった + response_so_far += repair_text + pipeline_info["final_status"] = "repair_no_code" + return postprocess_response(response_so_far), pipeline_info + else: + # リトライ上限到達: エラー出力を注入して続きを生成 + injection = f"\nError: {error_msg}\n\n" + response_so_far += injection + + full_context = prompt + response_so_far + context_ids = tokenizer( + full_context, return_tensors="pt", truncation=True, max_length=4096 + ).input_ids.to(model.device) + context_len = context_ids.shape[1] + + with torch.no_grad(): + outputs = model.generate( + input_ids=context_ids, + max_new_tokens=max_new_tokens // 2, + do_sample=False, + pad_token_id=tokenizer.pad_token_id, + eos_token_id=tokenizer.eos_token_id, + repetition_penalty=1.1, + ) + + continuation = tokenizer.decode( + outputs[0][context_len:], skip_special_tokens=True + ) + response_so_far += continuation + pipeline_info["final_status"] = "error_exhausted" + return postprocess_response(response_so_far), pipeline_info + + return postprocess_response(response_so_far), pipeline_info + + +def run_inference(model, tokenizer, test_data: list, max_new_tokens: int, + max_samples: int, code_exec_pipeline: bool = False, + max_retries: int = 2) -> list: + """全テストデータに対して推論を実行""" + samples = test_data if max_samples < 0 else test_data[:max_samples] + results = [] + total = len(samples) + + mode = "code execution pipeline" if code_exec_pipeline else "standard" + print(f"\nRunning inference on {total} samples... (mode: {mode})") + if code_exec_pipeline: + print(f" max_retries={max_retries} for self-repair on code execution error") + start_time = time.time() + + for i, sample in enumerate(samples): + t0 = time.time() + + if code_exec_pipeline: + response, pipeline_info = generate_with_code_exec( + model, tokenizer, sample["question"], max_new_tokens, max_retries + ) + else: + response = generate_response(model, tokenizer, sample["question"], max_new_tokens) + pipeline_info = None + + elapsed = time.time() - t0 + + # Perplexity: 参照回答に対するモデルのperplexity + ppl = compute_perplexity(model, tokenizer, sample["question"], sample["cot_answer"]) + + result = { + "id": i, + "question": sample["question"], + "reference_answer": sample["cot_answer"], + "model_response": response, + "inference_time_s": round(elapsed, 2), + "perplexity": round(ppl, 4), + } + if pipeline_info: + result["pipeline_info"] = pipeline_info + + results.append(result) + + if (i + 1) % 10 == 0 or (i + 1) == total: + elapsed_total = time.time() - start_time + eta = elapsed_total / (i + 1) * (total - i - 1) + status = "" + if pipeline_info: + status = f" [{pipeline_info['final_status']}]" + print(f" [{i+1}/{total}] {elapsed:.1f}s/sample{status}, ETA: {eta/60:.1f}min") + + total_time = time.time() - start_time + print(f"\nInference completed: {total_time/60:.1f}min total, {total_time/total:.1f}s/sample avg") + return results + + +def score_results(results: list, skip_code_exec: bool = False) -> list: + """推論結果を自動採点""" + print(f"\nScoring {len(results)} results...") + + for i, r in enumerate(results): + ref = r["reference_answer"] + resp = r["model_response"] + + # 1. Boxed Answer Exact Match + gold_answer = extract_boxed_answer(ref) + pred_answer = extract_boxed_answer(resp) + r["gold_boxed"] = gold_answer + r["pred_boxed"] = pred_answer + r["exact_match"] = math_equivalent(pred_answer, gold_answer) if (pred_answer and gold_answer) else False + + # 2. Code Execution + if skip_code_exec: + r["code_exec"] = {"has_code": False, "skipped": True} + else: + r["code_exec"] = check_code_execution(resp) + + # 3. Format Compliance + r["format"] = check_format_compliance(resp) + + if (i + 1) % 50 == 0: + print(f" [{i+1}/{len(results)}] scored") + + return results + + +def compute_summary(results: list) -> dict: + """集計サマリーを計算""" + total = len(results) + if total == 0: + return {} + + # Exact Match + em_correct = sum(1 for r in results if r["exact_match"]) + gold_extracted = sum(1 for r in results if r["gold_boxed"] is not None) + pred_extracted = sum(1 for r in results if r["pred_boxed"] is not None) + + # Code Execution + code_results = [r for r in results if r["code_exec"].get("has_code", False)] + code_success = sum(1 for r in code_results if r["code_exec"]["all_success"]) + + # Format Compliance + format_ok = sum(1 for r in results if r["format"]["format_ok"]) + has_boxed = sum(1 for r in results if r["format"]["has_boxed_answer"]) + no_bad_tokens = sum(1 for r in results if r["format"]["no_bad_tokens"]) + + # Perplexity + ppls = [r["perplexity"] for r in results if "perplexity" in r] + avg_ppl = sum(ppls) / len(ppls) if ppls else 0 + + summary = { + "total_samples": total, + "perplexity": { + "mean": round(avg_ppl, 4), + "min": round(min(ppls), 4) if ppls else 0, + "max": round(max(ppls), 4) if ppls else 0, + }, + "exact_match": { + "correct": em_correct, + "accuracy": round(em_correct / total * 100, 1), + "gold_extracted": gold_extracted, + "pred_extracted": pred_extracted, + }, + "code_execution": { + "samples_with_code": len(code_results), + "all_success": code_success, + "success_rate": round(code_success / len(code_results) * 100, 1) if code_results else 0, + }, + "format_compliance": { + "format_ok": format_ok, + "format_ok_rate": round(format_ok / total * 100, 1), + "has_boxed": has_boxed, + "has_boxed_rate": round(has_boxed / total * 100, 1), + "no_bad_tokens": no_bad_tokens, + }, + } + + # Code Execution Pipeline 統計 + pipeline_results = [r for r in results if "pipeline_info" in r] + if pipeline_results: + status_counts = {} + for r in pipeline_results: + status = r["pipeline_info"]["final_status"] + status_counts[status] = status_counts.get(status, 0) + 1 + total_retries = sum( + r["pipeline_info"]["total_attempts"] - 1 + for r in pipeline_results + if r["pipeline_info"]["total_attempts"] > 1 + ) + summary["code_exec_pipeline"] = { + "enabled": True, + "samples_processed": len(pipeline_results), + "status_counts": status_counts, + "total_retries": total_retries, + } + + return summary + + +def save_results(results: list, summary: dict, output_dir: str, model_label: str = None): + """結果を保存""" + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + label = f"_{model_label}" if model_label else "" + + # 1. JSON (全詳細) + json_path = output_path / f"eval_results{label}_{timestamp}.json" + with open(json_path, "w", encoding="utf-8") as f: + json.dump({"summary": summary, "results": results}, f, ensure_ascii=False, indent=2) + print(f" JSON: {json_path}") + + # 2. CSV (スコアシート) + csv_path = output_path / f"eval_scores{label}_{timestamp}.csv" + with open(csv_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow([ + "id", "question_preview", "exact_match", + "gold_boxed", "pred_boxed", + "perplexity", + "has_code", "code_success", + "format_ok", "has_boxed", "no_bad_tokens", + "inference_time_s", + ]) + for r in results: + writer.writerow([ + r["id"], + r["question"][:80], + int(r["exact_match"]), + r["gold_boxed"] or "", + r["pred_boxed"] or "", + r.get("perplexity", ""), + int(r["code_exec"].get("has_code", False)), + int(r["code_exec"].get("all_success", False)) if r["code_exec"].get("has_code") else "N/A", + int(r["format"]["format_ok"]), + int(r["format"]["has_boxed_answer"]), + int(r["format"]["no_bad_tokens"]), + r.get("inference_time_s", ""), + ]) + print(f" CSV: {csv_path}") + + # 3. サマリーレポート (テキスト) + report_path = output_path / f"eval_summary{label}_{timestamp}.txt" + with open(report_path, "w", encoding="utf-8") as f: + f.write("=" * 60 + "\n") + f.write("OpenMath Finetuning Evaluation Summary (Llama-3.1-Swallow-8B)\n") + if model_label: + f.write(f"Model: {model_label}\n") + f.write(f"Date: {timestamp}\n") + f.write("=" * 60 + "\n\n") + + s = summary + f.write(f"Total Samples: {s['total_samples']}\n\n") + + f.write("--- Perplexity ---\n") + f.write(f" Mean: {s['perplexity']['mean']}\n") + f.write(f" Min: {s['perplexity']['min']}, Max: {s['perplexity']['max']}\n\n") + + f.write("--- Exact Match (Primary Metric) ---\n") + f.write(f" Accuracy: {s['exact_match']['accuracy']}% ({s['exact_match']['correct']}/{s['total_samples']})\n") + f.write(f" Gold answer extracted: {s['exact_match']['gold_extracted']}/{s['total_samples']}\n") + f.write(f" Pred answer extracted: {s['exact_match']['pred_extracted']}/{s['total_samples']}\n\n") + + f.write("--- Code Execution ---\n") + f.write(f" Samples with code: {s['code_execution']['samples_with_code']}\n") + f.write(f" Success rate: {s['code_execution']['success_rate']}% ({s['code_execution']['all_success']}/{s['code_execution']['samples_with_code']})\n\n") + + f.write("--- Format Compliance ---\n") + f.write(f" Format OK: {s['format_compliance']['format_ok_rate']}% ({s['format_compliance']['format_ok']}/{s['total_samples']})\n") + f.write(f" Has boxed answer: {s['format_compliance']['has_boxed_rate']}%\n") + f.write(f" No bad tokens: {s['format_compliance']['no_bad_tokens']}/{s['total_samples']}\n\n") + + if "code_exec_pipeline" in s: + p = s["code_exec_pipeline"] + f.write("--- Code Execution Pipeline ---\n") + f.write(f" Samples processed: {p['samples_processed']}\n") + for status, count in sorted(p["status_counts"].items()): + f.write(f" {status}: {count}\n") + f.write(f" Total self-repair retries: {p['total_retries']}\n") + print(f" Report: {report_path}") + + return json_path, csv_path, report_path + + +def print_summary(summary: dict, model_label: str = None): + """サマリーをコンソールに表示""" + s = summary + print("\n" + "=" * 60) + title = f"EVALUATION SUMMARY [{model_label}]" if model_label else "EVALUATION SUMMARY" + print(title) + print("=" * 60) + print(f"Total Samples: {s['total_samples']}") + print() + print(f" Perplexity (mean): {s['perplexity']['mean']}") + print(f" Exact Match Accuracy: {s['exact_match']['accuracy']}%" + f" ({s['exact_match']['correct']}/{s['total_samples']})") + print(f" Code Execution Rate: {s['code_execution']['success_rate']}%" + f" ({s['code_execution']['all_success']}/{s['code_execution']['samples_with_code']} samples with code)") + print(f" Format Compliance: {s['format_compliance']['format_ok_rate']}%") + if "code_exec_pipeline" in s: + p = s["code_exec_pipeline"] + print() + print(" --- Code Execution Pipeline ---") + print(f" Samples processed: {p['samples_processed']}") + for status, count in sorted(p["status_counts"].items()): + print(f" {status}: {count}") + print(f" Total self-repair retries: {p['total_retries']}") + print("=" * 60) + + +def main(): + global SYSTEM_PROMPT + args = parse_args() + + # カスタムシステムプロンプトが指定された場合、グローバル変数を上書き + if args.system_prompt is not None: + SYSTEM_PROMPT = args.system_prompt + print(f"[INFO] Using custom system prompt ({len(SYSTEM_PROMPT)} chars)") + + print("=" * 60) + print("OpenMath Finetuning Evaluation (Llama-3.1-Swallow-8B)") + print(f"Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("=" * 60) + + if args.skip_inference: + # 既存の推論結果を読み込んでスコアリングのみ + results_path = args.results_json + if not results_path: + print("[ERROR] --results_json is required when using --skip_inference") + return + print(f"\nLoading existing results: {results_path}") + with open(results_path, encoding="utf-8") as f: + data = json.load(f) + results = data["results"] if "results" in data else data + else: + # テストデータ読み込み + print(f"\nTest data: {args.test_data}") + with open(args.test_data, encoding="utf-8") as f: + test_data = json.load(f) + print(f" {len(test_data)} samples loaded") + + # モデルロード + print(f"\nModel: {args.model_path}") + model, tokenizer = load_model(args.model_path, args.original_model_path, args.gpu_id) + + # 推論実行 + results = run_inference( + model, tokenizer, test_data, args.max_new_tokens, args.max_samples, + code_exec_pipeline=args.code_exec_pipeline, + max_retries=args.max_retries, + ) + + # GPUメモリ解放 + del model + gc.collect() + torch.cuda.empty_cache() + print(" [OK] Model unloaded") + + # スコアリング + results = score_results(results, skip_code_exec=args.skip_code_exec) + + # サマリー計算 + summary = compute_summary(results) + + # model_label の自動推定 + model_label = args.model_label + if model_label is None and not args.skip_inference: + if args.model_path == DEFAULT_ORIGINAL_MODEL: + model_label = "llama_original" + elif args.model_path == DEFAULT_FINETUNED_MODEL: + model_label = "llama_finetuned" + + # サマリーに metadata を追加 + summary["model_label"] = model_label + summary["model_path"] = args.model_path if not args.skip_inference else "(skip_inference)" + + # 保存 + print("\nSaving results...") + save_results(results, summary, args.output_dir, model_label) + + # サマリー表示 + print_summary(summary, model_label) + + +if __name__ == "__main__": + main() diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..3c1d049 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..0827079 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:268aef6511a866fd03104209ff6592ded093172c0cdaf8885ff9c97d62e32869 +size 17209958 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e8565ea --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 1000000000000000019884624838656, + "pad_token": "<|finetune_right_pad_id|>", + "tokenizer_class": "PreTrainedTokenizer" +}