67 lines
1.8 KiB
YAML
67 lines
1.8 KiB
YAML
|
model:
|
||
|
|
name: Mistral-Large-3-675B-Instruct-2512
|
||
|
|
model_type: mistral
|
||
|
|
|
||
|
|
paths:
|
||
|
|
checkpoint_dir: /home/mashun/vllm_project/checkpoints/Mistral-Large-3-675B-Instruct-2512
|
||
|
|
output_dir: /home/mashun/vllm_project/checkpoints/Mistral-Large-3-675B-Instruct-2512-BF16
|
||
|
|
|
||
|
|
runtime:
|
||
|
|
num_workers: 16
|
||
|
|
mode: dequant # dequant | dequant+quant | quant (only support bf16)
|
||
|
|
|
||
|
|
retry_files: ~
|
||
|
|
|
||
|
|
quant:
|
||
|
|
version: 2
|
||
|
|
format:
|
||
|
|
TN:
|
||
|
|
- "re:layers.\\d+.experts.*"
|
||
|
|
- "re:layers.\\d+.shared_experts.*"
|
||
|
|
NN: ~
|
||
|
|
batch_size: 64
|
||
|
|
block_size: 128
|
||
|
|
group_size: -1
|
||
|
|
symmetric: True
|
||
|
|
class:
|
||
|
|
int4:
|
||
|
|
- "re:layers.\\d+.experts.*"
|
||
|
|
int8:
|
||
|
|
- "re:layers.\\d+.shared_experts.*"
|
||
|
|
ignore:
|
||
|
|
- "re:^vision_language_adapter.*"
|
||
|
|
- "re:tok_embeddings.*"
|
||
|
|
- "re:pre_mm_projector_norm.*"
|
||
|
|
- "re:patch_merger.merging_layer.*"
|
||
|
|
- "output.weight"
|
||
|
|
- "norm.weight"
|
||
|
|
- "re:layers.\\d+.attention.kv_a_norm.*"
|
||
|
|
- "re:layers.\\d+.attention.q_a_norm.*"
|
||
|
|
- "re:layers.\\d+.attention.wkv_a_with_mqa.*"
|
||
|
|
- "re:layers.\\d+.attention.wq_a.*"
|
||
|
|
- "re:layers.\\d+.attention.wkv_b.*"
|
||
|
|
- "re:layers.\\d+.attention.wo.*"
|
||
|
|
- "re:layers.\\d+.attention.wq_b.*"
|
||
|
|
- "re:layers.\\d+.attention_norm.*"
|
||
|
|
- "re:layers.\\d+.ffn_norm.*"
|
||
|
|
- "re:layers.\\d+.gate.*"
|
||
|
|
- "re:layers.\\d+.feed_forward.*"
|
||
|
|
- "re:vision_encoder.*"
|
||
|
|
|
||
|
|
dequant:
|
||
|
|
ignore:
|
||
|
|
- "re:vision_language_adapter.*"
|
||
|
|
- "re:tok_embeddings.*"
|
||
|
|
- "re:pre_mm_projector_norm.*"
|
||
|
|
- "re:patch_merger.merging_layer.*"
|
||
|
|
- "output.weight"
|
||
|
|
- "norm.weight"
|
||
|
|
- "re:layers.\\d+.attention.kv_a_norm.*"
|
||
|
|
- "re:layers.\\d+.attention.q_a_norm.*"
|
||
|
|
- "re:layers.\\d+.attention.wkv_a_with_mqa.*"
|
||
|
|
- "re:layers.\\d+.attention.wq_a.*"
|
||
|
|
- "re:layers.\\d+.attention_norm.*"
|
||
|
|
- "re:layers.\\d+.ffn_norm.*"
|
||
|
|
- "re:layers.\\d+.gate.*"
|
||
|
|
- "re:vision_encoder.*"
|