"""
KeuralMoECausalLM — Keural Mixture-of-Experts Causal Language Model.

Bilingual Korean-English MoE LLM trained entirely from scratch.
Developed by MKD Corp AI Research, Republic of Korea.

Architecture:
  - 14.83B total parameters (~7.42B active per token)
  - 24 layers, hidden=4096, GQA 32/8 heads
  - 8 experts per layer, top-2 routing
  - Sliding window attention (512, alternating layers)
  - RoPE theta=500,000, context length=4096
  - Vocabulary: 131,074 tokens (131,072 SPM + <|im_start|> + <|im_end|>)

Load with trust_remote_code=True:
  from transformers import AutoModelForCausalLM, AutoTokenizer
  model = AutoModelForCausalLM.from_pretrained(
      "mkd-hossain/keural-sft3-final",
      torch_dtype="bfloat16",
      device_map="auto",
      trust_remote_code=True,
  )
"""

from transformers import MixtralForCausalLM
from transformers.utils import logging

try:
    from configuration_keural import KeuralConfig
except ImportError:
    from .configuration_keural import KeuralConfig

logger = logging.get_logger(__name__)


class KeuralMoECausalLM(MixtralForCausalLM):
    """
    Keural MoE Causal Language Model.
    Bilingual Korean-English 14.83B MoE LLM trained from scratch by MKD Corp AI Research.
    """

    config_class = KeuralConfig
    _keys_to_ignore_on_load_missing = ["lm_head.weight"]