44 lines
1.3 KiB
Python
44 lines
1.3 KiB
Python
"""
|
|
KeuralMoECausalLM — Keural Mixture-of-Experts Causal Language Model.
|
|
|
|
Bilingual Korean-English MoE LLM trained entirely from scratch.
|
|
Developed by MKD Corp AI Research, Republic of Korea.
|
|
|
|
Architecture:
|
|
- 14.83B total parameters (~7.42B active per token)
|
|
- 24 layers, hidden=4096, GQA 32/8 heads
|
|
- 8 experts per layer, top-2 routing
|
|
- Sliding window attention (512, alternating layers)
|
|
- RoPE theta=500,000, context length=4096
|
|
- Vocabulary: 131,074 tokens (131,072 SPM + <|im_start|> + <|im_end|>)
|
|
|
|
Load with trust_remote_code=True:
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
"mkd-hossain/keural-sft3-final",
|
|
torch_dtype="bfloat16",
|
|
device_map="auto",
|
|
trust_remote_code=True,
|
|
)
|
|
"""
|
|
|
|
from transformers import MixtralForCausalLM
|
|
from transformers.utils import logging
|
|
|
|
try:
|
|
from configuration_keural import KeuralConfig
|
|
except ImportError:
|
|
from .configuration_keural import KeuralConfig
|
|
|
|
logger = logging.get_logger(__name__)
|
|
|
|
|
|
class KeuralMoECausalLM(MixtralForCausalLM):
|
|
"""
|
|
Keural MoE Causal Language Model.
|
|
Bilingual Korean-English 14.83B MoE LLM trained from scratch by MKD Corp AI Research.
|
|
"""
|
|
|
|
config_class = KeuralConfig
|
|
_keys_to_ignore_on_load_missing = ["lm_head.weight"]
|