初始化项目，由ModelHub XC社区提供模型

Model: pathcosmos/frankenstallm
Source: Original Platform

source/tokenizer/convert_sp_to_hf.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+tokenizer/convert_sp_to_hf.py — SentencePiece 모델을 HuggingFace tokenizers.json으로 변환.
+
+prepare.py의 load_tokenizer()는 Tokenizer.from_file()을 사용하므로
+SentencePiece .model을 직접 읽지 못함 → HF tokenizers 포맷으로 변환 필요.
+
+Usage:
+    python tokenizer/convert_sp_to_hf.py \
+        --model tokenizer/korean_sp/tokenizer.model \
+        --output tokenizer/korean_sp/tokenizer.json
+
+Requirements:
+    pip install --break-system-packages sentencepiece tokenizers transformers
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def convert(model_path: Path, output_path: Path) -> None:
+    """SentencePiece Unigram 모델을 HuggingFace tokenizers.json으로 변환."""
+
+    # 방법 1: transformers의 XLNetTokenizer 계열 변환기 활용
+    # (더 완전한 변환, special token 처리 포함)
+    try:
+        from transformers.convert_slow_tokenizer import SpmConverter
+        from tokenizers import Tokenizer
+        from tokenizers.models import Unigram
+
+        print(f"변환 중: {model_path} → {output_path}")
+
+        # SpmConverter는 tokenizers 라이브러리의 Unigram 모델로 변환
+        # sentencepiece 모델 로드
+        import sentencepiece as spm
+        sp = spm.SentencePieceProcessor()
+        sp.load(str(model_path))
+
+        vocab_size = sp.vocab_size()
+        print(f"어휘 크기: {vocab_size:,}")
+
+        # Unigram vocab 추출: (piece, score) 목록
+        vocab: list[tuple[str, float]] = []
+        for i in range(vocab_size):
+            piece = sp.id_to_piece(i)
+            score = sp.get_score(i)
+            vocab.append((piece, score))
+
+        # HuggingFace Unigram 모델 생성
+        # unk_id 확인
+        unk_id = sp.unk_id()
+
+        tokenizer = Tokenizer(Unigram(vocab, unk_id=unk_id))
+
+        # Pre-tokenizer: Metaspace (SentencePiece 방식 — 공백을 ▁로 변환)
+        # tokenizers >= 0.14: add_prefix_space → prepend_scheme='always'
+        from tokenizers.pre_tokenizers import Metaspace
+        tokenizer.pre_tokenizer = Metaspace(replacement="▁", prepend_scheme="always")
+
+        # Decoder: Metaspace (역변환)
+        from tokenizers.decoders import Metaspace as MetaspaceDecoder
+        tokenizer.decoder = MetaspaceDecoder(replacement="▁", prepend_scheme="always")
+
+        # Special token 설정 (SP 모델과 동일한 ID)
+        from tokenizers import AddedToken
+        pad_id = sp.pad_id() if sp.pad_id() >= 0 else 0
+        bos_id = sp.bos_id() if sp.bos_id() >= 0 else 1
+        eos_id = sp.eos_id() if sp.eos_id() >= 0 else 2
+
+        tokenizer.add_special_tokens([
+            AddedToken("<pad>", special=True),
+            AddedToken("<s>", special=True),
+            AddedToken("</s>", special=True),
+            AddedToken("<unk>", special=True),
+        ])
+
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        tokenizer.save(str(output_path))
+
+        # 저장 후 검증
+        loaded = Tokenizer.from_file(str(output_path))
+        test_text = "안녕하세요, 한국어 언어 모델입니다."
+        encoded = loaded.encode(test_text)
+        print(f"\n검증 통과:")
+        print(f"  테스트 문자: {test_text!r}")
+        print(f"  토큰 수: {len(encoded.ids)}")
+        print(f"  토큰: {encoded.tokens[:15]}{'...' if len(encoded.tokens) > 15 else ''}")
+        print(f"\n저장 완료: {output_path}")
+
+    except ImportError as e:
+        print(f"ERROR: 필요한 라이브러리 없음: {e}", file=sys.stderr)
+        print("  pip install --break-system-packages sentencepiece tokenizers transformers", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"ERROR: 변환 실패: {e}", file=sys.stderr)
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="SentencePiece 모델 → HuggingFace tokenizers.json 변환",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--model",
+        type=Path,
+        required=True,
+        help="SentencePiece .model 파일 경로",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        required=True,
+        help="출력 tokenizers.json 경로",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    if not args.model.exists():
+        print(f"ERROR: 모델 파일 없음: {args.model}", file=sys.stderr)
+        sys.exit(1)
+    convert(args.model, args.output)
+
+
+if __name__ == "__main__":
+    main()

source/tokenizer/korean_sp/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edcf1eaa0a5ba871302ff42df9f80d1d0baa166ff2a57f4392c29145796bc7b2
+size 1424163

source/tokenizer/tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}

source/tokenizer/train_sp_tokenizer.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""
+tokenizer/train_sp_tokenizer.py — SentencePiece Unigram 한국어 토크나이저 학습.
+
+한국어 1음절(UTF-8 3바이트) = 1토큰이 되도록 Unigram 모델을 사용.
+character_coverage=0.9995로 한글 11,172 음절 전체 커버.
+
+Usage:
+    python tokenizer/train_sp_tokenizer.py \
+        --input "data/raw/namuwiki_ko/*.txt,data/raw/ko_wiki_0000.txt" \
+        --vocab_size 64000 \
+        --output_dir tokenizer/korean_sp
+
+Output:
+    tokenizer/korean_sp/tokenizer.model   (SentencePiece 모델)
+    tokenizer/korean_sp/tokenizer.vocab   (어휘 목록)
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import os
+import sys
+import tempfile
+from pathlib import Path
+
+
+def expand_inputs(input_spec: str) -> list[str]:
+    """콤마로 구분된 글로브 패턴들을 실제 파일 경로 목록으로 확장."""
+    files: list[str] = []
+    for pattern in input_spec.split(","):
+        pattern = pattern.strip()
+        if any(c in pattern for c in ("*", "?", "[")):
+            matched = sorted(glob.glob(pattern, recursive=True))
+            if not matched:
+                print(f"WARNING: 패턴에 일치하는 파일 없음: {pattern!r}", file=sys.stderr)
+            files.extend(matched)
+        else:
+            if Path(pattern).exists():
+                files.append(pattern)
+            else:
+                print(f"WARNING: 파일 없음: {pattern!r}", file=sys.stderr)
+    return files
+
+
+def train(
+    input_files: list[str],
+    output_dir: Path,
+    vocab_size: int,
+    num_threads: int,
+    input_sentence_size: int,
+) -> None:
+    try:
+        import sentencepiece as spm
+    except ImportError:
+        print(
+            "ERROR: sentencepiece가 설치되지 않음.\n"
+            "  pip install --break-system-packages sentencepiece",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    model_prefix = str(output_dir / "tokenizer")
+
+    print(f"입력 파일 수: {len(input_files)}")
+    for f in input_files[:5]:
+        print(f"  {f}")
+    if len(input_files) > 5:
+        print(f"  ... 외 {len(input_files) - 5}개")
+    print(f"어휘 크기: {vocab_size:,}")
+    print(f"출력 경로: {model_prefix}.model / .vocab")
+    print()
+
+    # SentencePiece는 파일 목록을 콤마로 구분된 단일 문자열로 받는다
+    input_str = ",".join(input_files)
+
+    spm.SentencePieceTrainer.train(
+        input=input_str,
+        model_prefix=model_prefix,
+        vocab_size=vocab_size,
+        model_type="unigram",               # BPE보다 한국어에 자연스러움
+        character_coverage=0.9995,           # 한글 11,172 음절 완전 커버
+        normalization_rule_name="nfkc",      # Unicode NFKC 정규화 (한국어 호환문자 통일)
+        pad_id=0,
+        bos_id=1,
+        eos_id=2,
+        unk_id=3,
+        pad_piece="<pad>",
+        bos_piece="<s>",
+        eos_piece="</s>",
+        unk_piece="<unk>",
+        user_defined_symbols=[],
+        num_threads=num_threads,
+        input_sentence_size=input_sentence_size,
+        shuffle_input_sentence=True,
+        # 학습 안정성
+        seed_sentencepiece_size=1_000_000,
+        shrinking_factor=0.75,
+        max_sentence_length=4096,
+    )
+
+    model_path = Path(f"{model_prefix}.model")
+    vocab_path = Path(f"{model_prefix}.vocab")
+
+    if model_path.exists():
+        size_mb = model_path.stat().st_size / 1e6
+        print(f"학습 완료!")
+        print(f"  모델: {model_path}  ({size_mb:.1f} MB)")
+        print(f"  어휘: {vocab_path}")
+        print()
+        print("다음 단계:")
+        print(f"  python tokenizer/convert_sp_to_hf.py \\")
+        print(f"    --model {model_path} \\")
+        print(f"    --output {output_dir}/tokenizer.json")
+    else:
+        print("ERROR: 학습 실패 — 출력 파일이 생성되지 않음", file=sys.stderr)
+        sys.exit(1)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="SentencePiece Unigram 한국어 토크나이저 학습",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--input",
+        required=True,
+        help="콤마로 구분된 파일/글로브 패턴 (예: 'data/raw/ko/*.txt,data/raw/wiki.txt')",
+    )
+    parser.add_argument(
+        "--vocab_size",
+        type=int,
+        default=64000,
+        help="어휘 크기",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=Path,
+        default=Path("tokenizer/korean_sp"),
+        help="모델 저장 디렉토리",
+    )
+    parser.add_argument(
+        "--num_threads",
+        type=int,
+        default=64,
+        help="학습에 사용할 CPU 스레드 수",
+    )
+    parser.add_argument(
+        "--input_sentence_size",
+        type=int,
+        default=10_000_000,
+        help="학습에 사용할 최대 문장 수 (0 = 무제한)",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    input_files = expand_inputs(args.input)
+    if not input_files:
+        print("ERROR: 입력 파일이 없습니다.", file=sys.stderr)
+        sys.exit(1)
+    train(
+        input_files=input_files,
+        output_dir=args.output_dir,
+        vocab_size=args.vocab_size,
+        num_threads=args.num_threads,
+        input_sentence_size=args.input_sentence_size,
+    )
+
+
+if __name__ == "__main__":
+    main()

source/tokenizer/train_tokenizer.py

@@ -0,0 +1,147 @@
+"""
+Train a Byte-Level BPE tokenizer on raw text files.
+
+The tokenizer is saved in two formats:
+  1. Native HuggingFace ``tokenizers`` format (vocab.json + merges.txt) inside
+     the output directory — for fast loading with ByteLevelBPETokenizer.
+  2. A ``tokenizer.json`` file (PreTrainedTokenizerFast) in the output directory
+     — for easy loading with transformers.AutoTokenizer.
+
+Usage:
+    python tokenizer/train_tokenizer.py \
+        --input  "data/raw/*.txt" \
+        --output  tokenizer/ \
+        --vocab_size 32000 \
+        --min_frequency 2
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import os
+import sys
+from pathlib import Path
+
+from tokenizers import AddedToken
+from tokenizers.implementations import ByteLevelBPETokenizer
+from transformers import PreTrainedTokenizerFast
+
+
+# ---------------------------------------------------------------------------
+# Special tokens
+# ---------------------------------------------------------------------------
+SPECIAL_TOKENS: list[str] = ["<pad>", "<s>", "</s>", "<unk>"]
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def find_input_files(pattern: str) -> list[str]:
+    """Resolve a glob pattern or a plain file path to a sorted list of paths."""
+    if any(c in pattern for c in ("*", "?", "[")):
+        files = sorted(glob.glob(pattern, recursive=True))
+    else:
+        files = [pattern] if Path(pattern).exists() else []
+    if not files:
+        raise FileNotFoundError(f"No files matched pattern: {pattern!r}")
+    return files
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Train a Byte-Level BPE tokenizer and save to disk."
+    )
+    parser.add_argument(
+        "--input",
+        required=True,
+        help='Glob pattern for training text files, e.g. "data/raw/*.txt"',
+    )
+    parser.add_argument(
+        "--output",
+        default="tokenizer/",
+        help="Output directory for the trained tokenizer (default: tokenizer/)",
+    )
+    parser.add_argument(
+        "--vocab_size",
+        type=int,
+        default=32000,
+        help="Target vocabulary size (default: 32000)",
+    )
+    parser.add_argument(
+        "--min_frequency",
+        type=int,
+        default=2,
+        help="Minimum frequency for a pair to be merged (default: 2)",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+
+    # ---- Discover input files ----
+    input_files = find_input_files(args.input)
+    print(f"Found {len(input_files)} training file(s).")
+
+    # ---- Create output directory ----
+    output_dir = Path(args.output)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # ---- Initialise tokenizer ----
+    tokenizer = ByteLevelBPETokenizer()
+
+    # ---- Train ----
+    print(
+        f"\nTraining BPE tokenizer | vocab_size={args.vocab_size} "
+        f"| min_frequency={args.min_frequency} ..."
+    )
+    tokenizer.train(
+        files=input_files,
+        vocab_size=args.vocab_size,
+        min_frequency=args.min_frequency,
+        special_tokens=SPECIAL_TOKENS,
+        show_progress=True,
+    )
+
+    # ---- Add special tokens explicitly (ensures they have the right IDs) ----
+    tokenizer.add_special_tokens(SPECIAL_TOKENS)
+
+    # ---- Save native format (vocab.json + merges.txt) ----
+    tokenizer.save_model(str(output_dir))
+    print(f"\nSaved vocab.json + merges.txt to: {output_dir}")
+
+    # ---- Wrap in PreTrainedTokenizerFast and save tokenizer.json ----
+    fast_tokenizer = PreTrainedTokenizerFast(
+        tokenizer_object=tokenizer._tokenizer,
+        bos_token="<s>",
+        eos_token="</s>",
+        unk_token="<unk>",
+        pad_token="<pad>",
+    )
+    tokenizer_json_path = output_dir / "tokenizer.json"
+    fast_tokenizer.save_pretrained(str(output_dir))
+    print(f"Saved PreTrainedTokenizerFast to: {output_dir}")
+    print(f"  -> tokenizer.json: {tokenizer_json_path}")
+
+    # ---- Stats ----
+    actual_vocab_size = tokenizer.get_vocab_size()
+    print("\n" + "=" * 50)
+    print("Tokenizer training statistics")
+    print("=" * 50)
+    print(f"  Training files  : {len(input_files):>10,}")
+    print(f"  Target vocab    : {args.vocab_size:>10,}")
+    print(f"  Actual vocab    : {actual_vocab_size:>10,}")
+    print(f"  Min frequency   : {args.min_frequency:>10,}")
+    print(f"  Special tokens  : {SPECIAL_TOKENS}")
+    print(f"  Output dir      : {output_dir.resolve()}")
+    print("=" * 50)
+
+
+if __name__ == "__main__":
+    main()