chore: update torch v2.5.1 (#1849)

python/sglang/srt/model_executor/model_runner.py

@@ -28,6 +28,7 @@ import torch
 import torch.nn as nn
 from vllm.config import DeviceConfig, LoadConfig
 from vllm.config import ModelConfig as VllmModelConfig
+from vllm.config import VllmConfig
 from vllm.distributed import (
     get_tp_group,
     init_distributed_environment,
@@ -59,6 +60,7 @@ from sglang.srt.utils import (
     enable_show_time_cost,
     get_available_gpu_memory,
     monkey_patch_vllm_dummy_weight_loader,
+    monkey_patch_vllm_model_config,
     monkey_patch_vllm_p2p_access_check,
 )
 
@@ -243,12 +245,14 @@ class ModelRunner:
 
         # Prepare the vllm model config
         monkey_patch_vllm_dummy_weight_loader()
+        monkey_patch_vllm_model_config()
         self.load_config = LoadConfig(
             load_format=self.server_args.load_format,
             download_dir=self.server_args.download_dir,
         )
         self.vllm_model_config = VllmModelConfig(
             model=self.server_args.model_path,
+            task="generate" if self.model_config.is_generation else "embedding",
             quantization=self.server_args.quantization,
             tokenizer=None,
             tokenizer_mode=None,
@@ -263,15 +267,17 @@ class ModelRunner:
             )
         self.dtype = self.vllm_model_config.dtype
 
+        self.vllm_config = VllmConfig()
+        self.vllm_config.model_config = self.vllm_model_config
+        self.vllm_config.load_config = self.load_config
+        self.vllm_config.device_config = DeviceConfig(self.device)
+        self.vllm_config.quant_config = VllmConfig._get_quantization_config(
+            self.vllm_config.model_config, self.vllm_config.load_config
+        )
+
         # Load the model
         self.model = get_model(
-            model_config=self.vllm_model_config,
-            load_config=self.load_config,
-            device_config=DeviceConfig(self.device),
-            parallel_config=None,
-            scheduler_config=None,
-            lora_config=None,
-            cache_config=None,
+            vllm_config=self.vllm_config,
         )
         self.sliding_window_size = (
             self.model.get_attention_sliding_window_size()
@@ -306,6 +312,7 @@ class ModelRunner:
             # TODO: Use a better method to check this
             vllm_model_config = VllmModelConfig(
                 model=model_path,
+                task="generate" if self.model_config.is_generation else "embedding",
                 quantization=self.server_args.quantization,
                 tokenizer=None,
                 tokenizer_mode=None,