EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[Fix, LoRA] fix LoRA with updates in main (#1545)

Browse Source
This commit is contained in:
Ying Sheng
2024-09-30 10:06:08 -07:00
committed by GitHub
parent 63ba2f8d7b
commit 0f4fb19bc8
5 changed files with 31 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
examples/runtime/lora.py
View File

@@ -1,5 +1,5 @@
# launch server
# python -m sglang.launch_server --model mistralai/Mistral-7B-Instruct-v0.3 --lora-paths /home/ying/test_lora /home/ying/test_lora_1 /home/ying/test_lora_2 lora3=/home/ying/test_lora_3 lora4=/home/ying/test_lora_4 --disable-radix --disable-cuda-graph --max-loras-per-batch 4
# python -m sglang.launch_server --model mistralai/Mistral-7B-Instruct-v0.3 --lora-paths /home/ying/test_lora lora1=/home/ying/test_lora_1 lora2=/home/ying/test_lora_2 --disable-radix --disable-cuda-graph --max-loras-per-batch 4
# send requests
# lora_path[i] specifies the LoRA used for text[i], so make sure they have the same length
@@ -22,12 +22,12 @@ json_data = {
"sampling_params": {"max_new_tokens": 32},
"lora_path": [
"/home/ying/test_lora",
"/home/ying/test_lora_1",
"/home/ying/test_lora_2",
"lora3",
"lora4",
"/home/ying/test_lora",
"/home/ying/test_lora_1",
"lora1",
"lora2",
"lora1",
"lora2",
None,
None,
],
}
response = requests.post(