74 lines
2.2 KiB
YAML
74 lines
2.2 KiB
YAML
|
model:
|
||
|
|
name: Qwen3.5-122B-A10B
|
||
|
|
model_type: qwen3_5_moe
|
||
|
|
|
||
|
|
paths:
|
||
|
|
checkpoint_dir: /home/mashun/vllm_project/checkpoints/Qwen3.5-122B-A10B
|
||
|
|
output_dir: /home/mashun/vllm_project/checkpoints/Qwen3.5-122B-A10B-w4a8
|
||
|
|
|
||
|
|
runtime:
|
||
|
|
num_workers: 16
|
||
|
|
mode: quant # dequant | dequant+quant | quant (only support bf16)
|
||
|
|
|
||
|
|
retry_files: ~
|
||
|
|
|
||
|
|
quant:
|
||
|
|
version: 2
|
||
|
|
format:
|
||
|
|
TN:
|
||
|
|
- "re:model.language_model.layers.\\d+.mlp.experts.gate_up_proj.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.mlp.experts.down_proj.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.self_attn.o_proj.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.self_attn.q_proj.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.self_attn.k_proj.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.self_attn.v_proj.*"
|
||
|
|
NN: ~
|
||
|
|
batch_size: 64
|
||
|
|
block_size: 128
|
||
|
|
group_size: -1
|
||
|
|
symmetric: True
|
||
|
|
class:
|
||
|
|
int4:
|
||
|
|
- "re:model.language_model.layers.\\d+.mlp.experts.gate_up_proj.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.mlp.experts.down_proj.*"
|
||
|
|
int8:
|
||
|
|
- "re:model.language_model.layers.\\d+.self_attn.o_proj.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.self_attn.q_proj.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.self_attn.k_proj.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.self_attn.v_proj.*"
|
||
|
|
ignore:
|
||
|
|
- "re:lm_head.*"
|
||
|
|
- "re:.*embed_tokens.*"
|
||
|
|
- "re:.*visual.*"
|
||
|
|
- "re:.*mtp.*"
|
||
|
|
- "re:.*norm.*"
|
||
|
|
- "re:.*linear_attn.*"
|
||
|
|
- "re:.*shared_expert.*"
|
||
|
|
- "re:model.language_model.layers.\\d+.mlp.gate\\..*"
|
||
|
|
- "re:.*bias.*"
|
||
|
|
|
||
|
|
dequant:
|
||
|
|
ignore:
|
||
|
|
- "re:.*mtp.fc.*"
|
||
|
|
- "re:.*mtp.layers.\\d+.mlp.gate.*"
|
||
|
|
- "re:.*mtp.layers.\\d+.mlp.shared_expert_gate.*"
|
||
|
|
- "re:.*visual.*"
|
||
|
|
- "re:.*shared_expert_gate.*"
|
||
|
|
- "re:.model.language_model.layers.\\d+.mlp.gate.*"
|
||
|
|
- "re:.*conv1d.*"
|
||
|
|
- "re:.*in_proj_a.*"
|
||
|
|
- "re:.*in_proj_b.*"
|
||
|
|
- "re:.*lm_head.*"
|
||
|
|
- "re:.*embed_tokens.*"
|
||
|
|
- "re:.*bias.*"
|
||
|
|
- "re:.*A_log.*"
|
||
|
|
- "re:.*model.language_model.layers.\\d+.linear_attn.norm.*"
|
||
|
|
- "re:.*input_layernorm.*"
|
||
|
|
- "re:.*post_attention_layernorm.*"
|
||
|
|
- "re:.*mtp.pre_fc_norm_embedding.*"
|
||
|
|
- "re:.*q_norm.*"
|
||
|
|
- "re:.*k_norm.*"
|
||
|
|
- "re:.*mtp.pre_fc_norm_hidden.*"
|
||
|
|
- "re:.*model.language_model.norm.*"
|
||
|
|
- "re:.*mtp.norm.*"
|