77 lines
2.5 KiB
YAML
77 lines
2.5 KiB
YAML
model:
|
|
name: Kimi-K2.5
|
|
model_type: kimik2
|
|
|
|
paths:
|
|
checkpoint_dir: /home/models/kimi/Kimi-K2.5
|
|
output_dir: /home/models/kimi/Kimi-K2.5-w4a8
|
|
|
|
runtime:
|
|
num_workers: 4
|
|
mode: dequant+quant # dequant | dequant+quant | quant (only support bf16)
|
|
|
|
retry_files: ~
|
|
|
|
quant:
|
|
version: 2
|
|
format:
|
|
TN:
|
|
- "re:.*model.layers.\\d+.mlp.experts.*"
|
|
- "re:.*model.layers.\\d+.self_attn.q_a_proj.*"
|
|
- "re:.*model.layers.\\d+.self_attn.q_b_proj.*"
|
|
- "re:.*model.layers.\\d+.self_attn.kv_a_proj_with_mqa.*"
|
|
- "re:.*model.layers.\\d+.self_attn.kv_b_proj.*"
|
|
- "re:.*model.layers.\\d+.self_attn.o_proj.*"
|
|
- "re:.*model.layers.\\d+.mlp.shared_experts.*"
|
|
- "re:.*model.layers.\\d+.mlp.gate_proj.*"
|
|
- "re:.*model.layers.\\d+.mlp.down_proj.*"
|
|
- "re:.*model.layers.\\d+.mlp.up_proj.*"
|
|
- "re:.*shared_experts.*"
|
|
NN: ~
|
|
batch_size: 64
|
|
block_size: 128
|
|
group_size: -1
|
|
symmetric: True
|
|
class:
|
|
int4:
|
|
- "re:.*model.layers.\\d+.mlp.experts.*"
|
|
int8:
|
|
- "re:.*model.layers.\\d+.self_attn.q_a_proj.*"
|
|
- "re:.*model.layers.\\d+.self_attn.q_b_proj.*"
|
|
- "re:.*model.layers.\\d+.self_attn.kv_a_proj_with_mqa.*"
|
|
- "re:.*model.layers.\\d+.self_attn.kv_b_proj.*"
|
|
- "re:.*model.layers.\\d+.self_attn.o_proj.*"
|
|
- "re:.*model.layers.\\d+.mlp.shared_experts.*"
|
|
- "re:.*model.layers.\\d+.mlp.gate_proj.*"
|
|
- "re:.*model.layers.\\d+.mlp.down_proj.*"
|
|
- "re:.*model.layers.\\d+.mlp.up_proj.*"
|
|
- "re:.*shared_experts.*"
|
|
ignore:
|
|
- "re:language_model.model.layers.\\d+.self_attn.q_a_layernorm.*"
|
|
- "re:language_model.model.layers.\\d+.self_attn.kv_a_layernorm.*"
|
|
- "re:language_model.model.layers.\\d+.input_layernorm.*"
|
|
- "re:language_model.model.layers.\\d+.post_attention_layernorm.*"
|
|
- "re:language_model.model.layers.\\d+.mlp.gate.weight"
|
|
- "re:language_model.model.layers.\\d+.mlp.gate.e_score_correction_bias"
|
|
- "re:language_model.model.layers.\\d+.self_attn.rotary_emb.inv_freq"
|
|
- "re:language_model.model.embed_tokens.*"
|
|
- "re:language_model.model.norm.*"
|
|
- "re:.*lm_head.*"
|
|
- "re:mm_projector.*"
|
|
- "re:vision_tower.*"
|
|
- "re:.*bias.*"
|
|
- "re:.*norm.*"
|
|
|
|
|
|
dequant:
|
|
ignore:
|
|
- "re:.*lm_head.*"
|
|
- "re:.*self_attn.*"
|
|
- "re:.*shared_experts.*"
|
|
- "re:.*mlp\\.(gate|up|gate_up|down)_proj.*"
|
|
- "re:.*bias.*"
|
|
- "re:.*norm.*"
|
|
- "re:language_model.model.layers.\\d+.mlp.gate.weight"
|
|
- "re:mm_projector.*"
|
|
- "re:vision_tower.*"
|
|
- "re:.*embed_tokens.*" |