43 lines
918 B
YAML
43 lines
918 B
YAML
model:
|
|
name: DeepSeek-R1
|
|
model_type: deepseek_v3
|
|
|
|
paths:
|
|
checkpoint_dir: /home/mashun/vllm_project/checkpoints/DeepSeek-R1-0528-FP8
|
|
output_dir: /home/mashun/vllm_project/checkpoints/DeepSeek-R1-0528
|
|
|
|
runtime:
|
|
num_workers: 8
|
|
mode: dequant # dequant | dequant+quant | quant (only support bf16)
|
|
|
|
retry_files: ~
|
|
|
|
quant:
|
|
version: 2
|
|
format:
|
|
TN: ~
|
|
NN: ~
|
|
batch_size: 64
|
|
block_size: 128
|
|
group_size: -1
|
|
symmetric: True
|
|
class:
|
|
int4: ~
|
|
int8: ~
|
|
ignore: ~
|
|
|
|
dequant:
|
|
ignore:
|
|
- "re:.*embed_tokens.*"
|
|
- "re:.*input_layernorm.*"
|
|
- "re:.*post_attention_layernorm.*"
|
|
- "re:.*kv_a_layernorm.*"
|
|
- "re:.*q_a_layernorm.*"
|
|
- "re:.*eh_proj.*"
|
|
- "re:.*enorm.*"
|
|
- "re:.*hnorm.*"
|
|
- "re:model.layers.\\d+.mlp.gate\\..*"
|
|
- "re:model.norm.*"
|
|
- "re:model.layers.\\d+.shared_head.norm.*"
|
|
- "re:.*lm_head.*"
|
|
- "re:model.layers.\\d+.shared_head.head.*" |