EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[Minor] Fix styles for overlap mode (#2068)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-11-17 19:49:20 -08:00
committed by GitHub
parent 8c280cee55
commit a9e90b4bce
4 changed files with 8 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
python/sglang/srt/managers/tp_worker_overlap_thread.py
View File

@@ -26,7 +26,6 @@ import torch
from sglang.srt.managers.io_struct import UpdateWeightReqInput
from sglang.srt.managers.schedule_batch import ModelWorkerBatch
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.server_args import ServerArgs
logger = logging.getLogger(__name__)
@@ -176,16 +175,8 @@ class TpModelWorkerClient:
) % self.future_token_ids_limit
return None, future_next_token_ids
def forward_batch_embedding(self, model_worker_batch: ModelWorkerBatch):
forward_batch = ForwardBatch.init_new(model_worker_batch, self.model_runner)
logits_output = self.model_runner.forward(forward_batch)
embeddings = logits_output.embeddings
return embeddings
def update_weights(self, recv_req: UpdateWeightReqInput):
success, message = self.model_runner.update_weights(
recv_req.model_path, recv_req.load_format
)
success, message = self.worker.update_weights(recv_req)
return success, message
def __delete__(self):