model:
  name: GLM-4.7
  model_type: glm4_moe

paths:
  checkpoint_dir: /home/mashun/vllm_project/checkpoints/GLM-4.7-FP8
  output_dir: /home/mashun/vllm_project/checkpoints/GLM-4.7-w4a8

runtime:
  num_workers: 16
  mode: dequant+quant # dequant | dequant+quant | quant (only support bf16)

retry_files: ~

quant:
  version: 2
  format: 
    TN:
      - "re:model.layers.\\d+.mlp.experts.\\d+.down_proj.*"
      - "re:model.layers.\\d+.mlp.experts.\\d+.up_proj.*"
      - "re:model.layers.\\d+.mlp.experts.\\d+.gate_proj.*"
    NN: ~
  batch_size: 64
  block_size: 128
  group_size: -1
  symmetric: True
  class:
    int4:
      - "re:model.layers.\\d+.mlp.experts.\\d+.down_proj.*"
      - "re:model.layers.\\d+.mlp.experts.\\d+.up_proj.*"
      - "re:model.layers.\\d+.mlp.experts.\\d+.gate_proj.*"
    int8: ~
  ignore: 
    - "re:.*self_attn.*"
    - "re:.*bias.*"
    - "re:.*mlp.gate.*"
    - "re:.*norm.*"
    - "re:.*lm_head.*"
    - "re:.*eh_proj.*"
    - "re:.*embed_tokens.*"
    - "re:.*shared_head.*"
    - "re:model.layers.\\d+.mlp.shared_experts.*"
    - "re:model.layers.\\d+.mlp.down_proj.*"
    - "re:model.layers.\\d+.mlp.up_proj.*"
    - "re:model.layers.\\d+.mlp.proj_proj.*"
   
dequant:
  ignore:
    - "re:.*bias"
    - "re:.*mlp.gate\\..*"
    - "re:.*norm.*"
    - "lm_head.weight"
    - "re:.*eh_proj.*"
    - "re:.*embed_tokens.weight"
    - "re:.*shared_head.*"