Update model_loader deps and qqq quantization deps (#2220) (#2318)

Co-authored-by: HandH1998 <1335248067@qq.com>
2024-12-02 23:22:13 +08:00
parent 33deca81b5
commit 85e1a6f3aa
58 changed files with 2363 additions and 366 deletions
--- a/python/sglang/bench_one_batch.py
+++ b/python/sglang/bench_one_batch.py
@@ -111,8 +111,12 @@ def load_model(server_args, port_args, tp_rank):
    model_config = ModelConfig(
        server_args.model_path,
        trust_remote_code=server_args.trust_remote_code,
+        revision=server_args.revision,
        context_length=server_args.context_length,
        model_override_args=server_args.json_model_override_args,
+        is_embedding=server_args.is_embedding,
+        dtype=server_args.dtype,
+        quantization=server_args.quantization,
    )
    model_runner = ModelRunner(
        model_config=model_config,