EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[Feature] add multi-rank support for Lora (#4492)

Browse Source 
Co-authored-by: rudy152 <czh1137892874@gmail.com>
This commit is contained in:
chaobo jia
2025-03-29 00:38:44 +08:00
committed by GitHub
parent 6dea5c96bf
commit ef9a378a20
16 changed files with 292 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
test/srt/models/lora/test_lora.py
View File

@@ -29,7 +29,7 @@ LORA_SETS = [
# {"base": "Qwen/Qwen2.5-14B-Instruct", "loras": ["mssongit/Qwen2.5-14B-SFT-LoRA"]},
# {"base": "mistralai/Mistral-7B-Instruct-v0.3", "loras": ["/home/ying/test_lora"]},
# {
# "base": "mistralai/Mistral-7B-Instruct-v0.3",
# "base": "mistralai/Mistral-7B-Instruct-v0.3",
# "loras": [
# "/home/ying/test_lora",
# "/home/ying/test_lora_1",
@@ -176,9 +176,11 @@ class TestLoRA(CustomTestCase):
print(f"{srt_no_lora_outputs.output_strs=}")
print(f"{srt_outputs_lora_path_none.output_strs=}")
for i in range(len(prompts)):
assert srt_outputs.output_strs[i].strip(" ") == hf_outputs.output_strs[i], (
assert srt_outputs.output_strs[i].strip(" ") == hf_outputs.output_strs[
i
].strip(" "), (
srt_outputs.output_strs[i].strip(" "),
hf_outputs.output_strs[i],
hf_outputs.output_strs[i].strip(" "),
)
assert (
srt_no_lora_outputs.output_strs[i].strip(" ")
@@ -187,7 +189,7 @@ class TestLoRA(CustomTestCase):
srt_no_lora_outputs.output_strs[i].strip(" "),
hf_no_lora_outputs.output_strs[i],
)
assert srt_outputs_lora_path_none == srt_no_lora_outputs
# assert srt_outputs_lora_path_none == srt_no_lora_outputs
def serving(self, prompts, lora_set, tp_size, torch_dtype, max_new_tokens):
print("=================== testing serving =======================")
@@ -287,7 +289,7 @@ class TestLoRA(CustomTestCase):
tp_size = 1
max_new_tokens = 32
self.inference(PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens)
self.serving(PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens)
# self.serving(PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens)
# self.base_inference(
# PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens
# )

135
test/srt/models/lora/test_multi_lora_backend.py
View File

@@ -19,17 +19,35 @@ from typing import List
import torch
from utils import BACKENDS, TORCH_DTYPES, LoRAAdaptor, LoRAModelCase
from sglang.test.test_utils import CustomTestCase, is_in_ci
from sglang.test.runners import HFRunner, SRTRunner
from sglang.test.test_utils import CustomTestCase, calculate_rouge_l, is_in_ci
MULTI_LORA_MODELS = [
# multi-rank case
LoRAModelCase(
base="meta-llama/Llama-2-7b-hf",
adaptors=[
LoRAAdaptor(
name="winddude/wizardLM-LlaMA-LoRA-7B",
prefill_tolerance=1e-1,
),
LoRAAdaptor(
name="RuterNorway/Llama-2-7b-chat-norwegian-LoRa",
prefill_tolerance=3e-1,
),
],
max_loras_per_batch=2,
),
LoRAModelCase(
base="meta-llama/Llama-3.1-8B-Instruct",
adaptors=[
LoRAAdaptor(
name="algoprog/fact-generation-llama-3.1-8b-instruct-lora",
prefill_tolerance=1e-1,
),
LoRAAdaptor(
name="some-org/another-lora-adaptor",
name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
prefill_tolerance=1e-1,
),
],
max_loras_per_batch=2,
@@ -64,6 +82,7 @@ class TestMultiLoRABackend(CustomTestCase):
The multi-LoRA backend test functionality is not supported yet.
This function uses all prompts at once and prints a message indicating that support is pending.
"""
base_path = model_case.base
adaptor_names = [adaptor.name for adaptor in model_case.adaptors]
print(
f"\n========== Testing multi-LoRA backend '{backend}' for base '{model_case.base}' --- "
@@ -72,6 +91,118 @@ class TestMultiLoRABackend(CustomTestCase):
print(
"run_backend_batch: Multi-LoRA backend test functionality is pending support."
)
with SRTRunner(
base_path,
torch_dtype=torch_dtype,
model_type="generation",
tp_size=model_case.tp_size,
lora_paths=[adaptor.name for adaptor in model_case.adaptors],
max_loras_per_batch=model_case.max_loras_per_batch,
lora_backend=backend,
disable_cuda_graph=True,
disable_radix_cache=True,
mem_fraction_static=0.88,
) as srt_runner:
srt_outputs = srt_runner.forward(
prompts, max_new_tokens=max_new_tokens, lora_paths=adaptor_names
)
with HFRunner(
base_path, torch_dtype=torch_dtype, model_type="generation"
) as hf_runner:
hf_outputs = hf_runner.forward(
prompts, max_new_tokens=max_new_tokens, lora_paths=adaptor_names
)
with SRTRunner(
base_path,
torch_dtype=torch_dtype,
model_type="generation",
tp_size=model_case.tp_size,
mem_fraction_static=0.88,
) as srt_runner:
srt_no_lora_outputs = srt_runner.forward(
prompts, max_new_tokens=max_new_tokens
)
with HFRunner(
base_path,
torch_dtype=torch_dtype,
model_type="generation",
) as hf_runner:
hf_no_lora_outputs = hf_runner.forward(
prompts, max_new_tokens=max_new_tokens
)
# Compare prefill stage logprobs (HF vs SRTRunner with LoRA)
for i in range(len(prompts)):
adaptor = model_case.adaptors[i]
# Use individual adapter tolerances if set, otherwise use model defaults
prefill_tol = (
adaptor.prefill_tolerance
if adaptor.prefill_tolerance is not None
else model_case.prefill_tolerance
)
decode_tol = (
adaptor.decode_tolerance
if adaptor.decode_tolerance is not None
else model_case.decode_tolerance
)
rouge_tol = (
adaptor.rouge_l_tolerance
if adaptor.rouge_l_tolerance is not None
else model_case.rouge_l_tolerance
)
# Compare prefill stage logprobs (HF vs SRTRunner with LoRA)
hf_prefill = torch.tensor(hf_outputs.top_input_logprobs[i])
srt_prefill = torch.tensor(srt_outputs.top_input_logprobs[i])
max_prefill_diff = torch.max(torch.abs(hf_prefill - srt_prefill))
print("Max prefill diff (HF vs SRT):", max_prefill_diff)
# Compare decode stage logprobs
hf_decode = torch.tensor(hf_outputs.top_output_logprobs[i])
srt_decode = torch.tensor(srt_outputs.top_output_logprobs[i])
max_decode_diff = torch.max(torch.abs(hf_decode - srt_decode))
print("Max decode diff (HF vs SRT):", max_decode_diff)
srt_output_str = srt_outputs.output_strs[i].strip()
hf_output_str = hf_outputs.output_strs[i].strip()
rouge_score = calculate_rouge_l([srt_output_str], [hf_output_str])[0]
print("ROUGE-L score:", rouge_score)
print("SRT output:", srt_output_str)
print("HF output:", hf_output_str)
# Additional: compare prefill outputs between base model (no LoRA) and LoRA model for reference
hf_no_lora_prefill = torch.tensor(hf_no_lora_outputs.top_input_logprobs[i])
srt_no_lora_prefill = torch.tensor(
srt_no_lora_outputs.top_input_logprobs[i]
)
print(
"Max diff (SRT base vs SRT LoRA prefill):",
torch.max(torch.abs(srt_no_lora_prefill - srt_prefill)),
)
print(
"Max diff (HF base vs HF LoRA prefill):",
torch.max(torch.abs(hf_no_lora_prefill - hf_prefill)),
)
if hf_prefill.shape[0] <= 100:
assert torch.all(torch.abs(hf_prefill - srt_prefill) < prefill_tol), (
f"Prefill logprobs mismatch for base '{base_path}', adaptor '{adaptor_names}', "
f"backend '{backend}', prompt: '{prompts[0][:50]}...'"
)
if hf_decode.shape[0] <= 100:
assert torch.all(torch.abs(hf_decode - srt_decode) < decode_tol), (
f"Decode logprobs mismatch for base '{base_path}', adaptor '{adaptor_names}', "
f"backend '{backend}', prompt: '{prompts[0][:50]}...'"
)
if rouge_score < rouge_tol:
raise AssertionError(
f"ROUGE-L score {rouge_score} below tolerance {rouge_tol} "
f"for base '{base_path}', adaptor '{adaptor_names}', backend '{backend}', prompt: '{prompts[0][:50]}...'"
)
def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]):
for model_case in model_cases:

4
test/srt/models/lora/utils.py
View File

@@ -31,8 +31,8 @@ class LoRAModelCase:
base: str
adaptors: List[LoRAAdaptor]
tp_size: int = 1
prefill_tolerance: float = 5e-2
decode_tolerance: float = 5e-2
prefill_tolerance: float = 1e-1
decode_tolerance: float = 1e-1
rouge_l_tolerance: float = 1.0
max_loras_per_batch: int = 1
skip_long_prompt: bool = False

2
test/srt/run_suite.py
View File

@@ -15,7 +15,7 @@ suites = {
"per-commit": [
TestFile("models/lora/test_lora.py", 76),
TestFile("models/lora/test_lora_backend.py", 420),
TestFile("models/lora/test_multi_lora_backend.py", 1),
TestFile("models/lora/test_multi_lora_backend.py", 144),
TestFile("models/test_embedding_models.py", 119),
TestFile("models/test_generation_models.py", 103),
TestFile("models/test_grok_models.py", 60),