EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix llava & Fix multiprocessing

Browse Source
This commit is contained in:
Lianmin Zheng
2024-01-24 10:35:31 +00:00
parent c6576e820c
commit bef0b35902
4 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/quick_start/srt_example_llava.py
View File

@@ -7,10 +7,8 @@ def image_qa(s, image_path, question):
s += sgl.assistant(sgl.gen("answer")) s += sgl.assistant(sgl.gen("answer"))
# runtime = sgl.Runtime(model_path="liuhaotian/llava-v1.5-7b", runtime = sgl.Runtime(model_path="liuhaotian/llava-v1.5-7b",
# tokenizer_path="llava-hf/llava-1.5-7b-hf") tokenizer_path="llava-hf/llava-1.5-7b-hf")
runtime = sgl.Runtime(model_path="llava-internal/llava-v1.6-7b-hd-224px_3x2-preview-20230103",
tokenizer_path="llava-internal/llava-v1.6-7b-hd-224px_3x2-preview-20230103-tokenizer")
sgl.set_default_backend(runtime) sgl.set_default_backend(runtime)

5
python/sglang/srt/managers/tokenizer_manager.py
View File

@@ -1,6 +1,7 @@
import asyncio import asyncio
import concurrent.futures import concurrent.futures
import dataclasses import dataclasses
import multiprocessing as mp
import os import os
from typing import List from typing import List
@@ -101,7 +102,9 @@ class TokenizerManager:
self.tokenizer = self.processor.tokenizer self.tokenizer = self.processor.tokenizer
os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["TOKENIZERS_PARALLELISM"] = "false"
self.executor = concurrent.futures.ProcessPoolExecutor( self.executor = concurrent.futures.ProcessPoolExecutor(
initializer=init_global_processor, initargs=(server_args,) initializer=init_global_processor,
mp_context=mp.get_context("fork"),
initargs=(server_args,),
) )
else: else:
self.tokenizer = get_tokenizer( self.tokenizer = get_tokenizer(

5
python/sglang/srt/models/llava.py
View File

@@ -34,9 +34,10 @@ class LlavaLlamaForCausalLM(nn.Module):
self.config.text_config.hidden_size = config.hidden_size self.config.text_config.hidden_size = config.hidden_size
self.multi_modal_projector = LlavaMultiModalProjector(config) self.multi_modal_projector = LlavaMultiModalProjector(config)
self.language_model = LlamaForCausalLM(config, linear_method) self.language_model = LlamaForCausalLM(config, linear_method)
if "unpad" in getattr(config, "mm_patch_merge_type"): if "unpad" in getattr(config, "mm_patch_merge_type", ""):
self.language_model.model.image_newline = nn.Parameter( self.language_model.model.image_newline = nn.Parameter(
torch.empty(config.text_config.hidden_size, dtype=torch.float16)) torch.empty(config.text_config.hidden_size, dtype=torch.float16)
)
def pad_input_ids(self, input_ids, pad_value, pt_shape=None, image_size=None): def pad_input_ids(self, input_ids, pad_value, pt_shape=None, image_size=None):
new_image_feature_len = self.image_feature_len new_image_feature_len = self.image_feature_len

2
python/sglang/srt/server.py
View File

@@ -425,8 +425,6 @@ class Runtime:
random_seed=random_seed, random_seed=random_seed,
log_level=log_level, log_level=log_level,
) )
import torch
torch.multiprocessing.set_start_method("spawn", force=True)
self.url = self.server_args.url() self.url = self.server_args.url()
self.generate_url = ( self.generate_url = (