62 lines
1.4 KiB
YAML
62 lines
1.4 KiB
YAML
model:
|
|
name: Step-3.5-Flash
|
|
model_type: step3p5
|
|
|
|
paths:
|
|
checkpoint_dir: /home/mashun/vllm_project/checkpoints/Step-3.5-Flash
|
|
output_dir: /home/mashun/vllm_project/checkpoints/Step-3.5-Flash-w4a8
|
|
|
|
runtime:
|
|
num_workers: 16
|
|
mode: quant # dequant | dequant+quant | quant (only support bf16)
|
|
|
|
retry_files: ~
|
|
|
|
quant:
|
|
version: 2
|
|
format:
|
|
TN:
|
|
- "re:.*moe.down_proj.*"
|
|
- "re:.*moe.gate_proj.*"
|
|
- "re:.*moe.up_proj.*"
|
|
- "re:.*self_attn.g_proj.*"
|
|
- "re:.*self_attn.k_proj.*"
|
|
- "re:.*self_attn.o_proj.*"
|
|
- "re:.*self_attn.q_proj.*"
|
|
- "re:.*self_attn.v_proj.*"
|
|
- "re:.*share_expert.*"
|
|
- "re:.*mlp.down_proj.*"
|
|
- "re:.*mlp.up_proj.*"
|
|
- "re:.*mlp.gate_proj.*"
|
|
NN: ~
|
|
batch_size: 64
|
|
block_size: 128
|
|
group_size: -1
|
|
symmetric: True
|
|
class:
|
|
int4:
|
|
- "re:.*moe.down_proj.*"
|
|
- "re:.*moe.gate_proj.*"
|
|
- "re:.*moe.up_proj.*"
|
|
int8:
|
|
- "re:.*self_attn.g_proj.*"
|
|
- "re:.*self_attn.k_proj.*"
|
|
- "re:.*self_attn.o_proj.*"
|
|
- "re:.*self_attn.q_proj.*"
|
|
- "re:.*self_attn.v_proj.*"
|
|
- "re:.*mlp.down_proj.*"
|
|
- "re:.*mlp.up_proj.*"
|
|
- "re:.*mlp.gate_proj.*"
|
|
ignore:
|
|
- "re:.*norm.*"
|
|
- "re:.*bias.*"
|
|
- "re:.*moe.gate\\..*"
|
|
- "re:.*eh_proj.*"
|
|
- "re:.*transformer.*"
|
|
- "re:.*lm_head.*"
|
|
- "re:.*embed_tokens.*"
|
|
- "re:.*share_expert.*"
|
|
|
|
|
|
dequant:
|
|
ignore: ~ |