初始化项目，由ModelHub XC社区提供模型
Model: cooler8/yejin-korean-3b-v2-base Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
35
README.md
Normal file
35
README.md
Normal file
@@ -0,0 +1,35 @@
|
||||
---
|
||||
language: [ko, en]
|
||||
license: apache-2.0
|
||||
library_name: transformers
|
||||
pipeline_tag: text-generation
|
||||
tags: [korean, pretrained, from-scratch, qwen3-architecture, causal-lm]
|
||||
---
|
||||
|
||||
# yejin-korean-3b-v2-base
|
||||
|
||||
**100% from-scratch** 사전학습된 한국어 3B 파운데이션 모델. 외부 공개 모델 가중치를 전혀 사용하지 않았습니다.
|
||||
|
||||
- 파라미터: 3.02B (hidden 3072, 28 layers, 24 heads, 8 KV heads, GQA 3:1)
|
||||
- 컨텍스트 4096, RoPE theta 500000, QK-Norm, tied embeddings
|
||||
- 토크나이저: 독자 64K Byte-level BPE (한국어 특화)
|
||||
- 학습: 8x NVIDIA H200, 34000 steps, 약 73B 토큰 (155GB 한국어 코퍼스: AI Hub, CulturaX, 위키백과, 교과서 등)
|
||||
- 학습 손실: ~2.0
|
||||
|
||||
## HF 아키텍처 클래스에 관하여
|
||||
학습 모델은 QK-Norm(헤드별 RMSNorm, RoPE 이전) 을 사용합니다. HF `LlamaForCausalLM` 에는 QK-Norm이 없어
|
||||
동일 구조를 지원하는 `Qwen3ForCausalLM` 클래스로 로드합니다. **가중치는 Qwen 과 무관한 독자 학습 결과**이며,
|
||||
클래스는 모델 구조(연산 그래프) 호환을 위해서만 사용됩니다. vLLM / llama.cpp 에서 그대로 로드 가능합니다.
|
||||
|
||||
## 사용
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
tok = AutoTokenizer.from_pretrained("cooler8/yejin-korean-3b-v2-base")
|
||||
model = AutoModelForCausalLM.from_pretrained("cooler8/yejin-korean-3b-v2-base", torch_dtype="bfloat16", device_map="auto")
|
||||
ids = tok("대한민국의 수도는", return_tensors="pt").to(model.device)
|
||||
print(tok.decode(model.generate(**ids, max_new_tokens=50)[0]))
|
||||
```
|
||||
|
||||
이 모델은 base 모델입니다 (지시문 튜닝 없음). SFT/DPO 버전: `cooler8/yejin-korean-3b-v2-sft`, `cooler8/yejin-korean-3b-v2-dpo`.
|
||||
|
||||
변환: step 34000 체크포인트, 2026-09-12 02:27 UTC
|
||||
33
config.json
Normal file
33
config.json
Normal file
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen3ForCausalLM"
|
||||
],
|
||||
"model_type": "qwen3",
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": [
|
||||
7,
|
||||
2
|
||||
],
|
||||
"pad_token_id": 3,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 3072,
|
||||
"intermediate_size": 8192,
|
||||
"num_hidden_layers": 28,
|
||||
"num_attention_heads": 24,
|
||||
"num_key_value_heads": 8,
|
||||
"head_dim": 128,
|
||||
"max_position_embeddings": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"rope_theta": 500000.0,
|
||||
"rope_scaling": null,
|
||||
"sliding_window": null,
|
||||
"use_sliding_window": false,
|
||||
"max_window_layers": 28,
|
||||
"tie_word_embeddings": true,
|
||||
"torch_dtype": "bfloat16",
|
||||
"use_cache": true,
|
||||
"vocab_size": 64000
|
||||
}
|
||||
13
generation_config.json
Normal file
13
generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": [
|
||||
7,
|
||||
2
|
||||
],
|
||||
"pad_token_id": 3,
|
||||
"do_sample": true,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.9,
|
||||
"repetition_penalty": 1.1,
|
||||
"max_new_tokens": 512
|
||||
}
|
||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a43bcf318e8d2ba8cbd07f832e2ca620156b0e39c8e708b470e61aff3d98f3c3
|
||||
size 6030760784
|
||||
36
special_tokens_map.json
Normal file
36
special_tokens_map.json
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
{
|
||||
"content": "<|system|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<|user|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<|assistant|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<|end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
],
|
||||
"bos_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"pad_token": "<pad>",
|
||||
"unk_token": "<unk>"
|
||||
}
|
||||
1
tokenizer.json
Normal file
1
tokenizer.json
Normal file
File diff suppressed because one or more lines are too long
85
tokenizer_config.json
Normal file
85
tokenizer_config.json
Normal file
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"3": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"4": {
|
||||
"content": "<|system|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"5": {
|
||||
"content": "<|user|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"6": {
|
||||
"content": "<|assistant|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"7": {
|
||||
"content": "<|end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|system|>",
|
||||
"<|user|>",
|
||||
"<|assistant|>",
|
||||
"<|end|>"
|
||||
],
|
||||
"bos_token": "<s>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "</s>",
|
||||
"extra_special_tokens": {},
|
||||
"model_max_length": 4096,
|
||||
"pad_token": "<pad>",
|
||||
"tokenizer_class": "PreTrainedTokenizerFast",
|
||||
"unk_token": "<unk>",
|
||||
"chat_template": "{{ bos_token }}{% for message in messages %}{{ '<|' + message['role'] + '|>\\n' + message['content'] + '<|end|>\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\\n' }}{% endif %}",
|
||||
"add_bos_token": true,
|
||||
"add_eos_token": false
|
||||
}
|
||||
36
training_stats.json
Normal file
36
training_stats.json
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"vocab_size": 64000,
|
||||
"total_texts": 6179192,
|
||||
"corpus_size_bytes": 17442976221,
|
||||
"training_time_seconds": 1414.3693861961365,
|
||||
"data_sources": {
|
||||
"01_논문_학술": 500042,
|
||||
"02_도서_말뭉치": 500003,
|
||||
"03_전문분야": 509076,
|
||||
"04_웹_뉴스": 501738,
|
||||
"05_법률_행정": 500000,
|
||||
"06_의료_건강": 500000,
|
||||
"07_교육_지식": 214238,
|
||||
"culturax_korean": 200000,
|
||||
"ko_alpaca_instructions": 21149,
|
||||
"korean_textbooks_claude_evol": 200000,
|
||||
"korean_textbooks_code_alpaca": 64006,
|
||||
"korean_textbooks_instructions": 200000,
|
||||
"korean_textbooks_mmlu_all": 97697,
|
||||
"korean_textbooks_tiny": 200000,
|
||||
"korean_textbooks_wikidata": 127470,
|
||||
"wikipedia_korean_2024": 200000,
|
||||
"ko_alpaca_v1.1a": 21149,
|
||||
"ko_orca": 34163,
|
||||
"korean_webtext": 200000,
|
||||
"mc4_korean": 200000,
|
||||
"namuwiki": 200000,
|
||||
"open_ko_instructions": 200000,
|
||||
"cosmopedia_stories_science": 200000,
|
||||
"magpie_reasoning_150k": 148703,
|
||||
"math_instruct": 200000,
|
||||
"python_code_instructions_18k": 18609
|
||||
},
|
||||
"date": "2026-08-29 19:30:06",
|
||||
"repo_id": "cooler8/yejin-korean-tokenizer-64k"
|
||||
}
|
||||
Reference in New Issue
Block a user