Enable Nvidia's ModelOpt fp8 quantized models (#2535)

2025-01-06 14:54:52 -08:00
parent b8574f6953
commit 287427e2e6
5 changed files with 185 additions and 0 deletions
--- a/python/sglang/srt/model_executor/model_runner.py
+++ b/python/sglang/srt/model_executor/model_runner.py
@@ -150,6 +150,7 @@ class ModelRunner:
                "enable_nan_detection": server_args.enable_nan_detection,
                "enable_dp_attention": server_args.enable_dp_attention,
                "enable_ep_moe": server_args.enable_ep_moe,
+                "modelopt_config": server_args.modelopt_config,
            }
        )