[Fix] Fix bugs and refactor codes in lora for better scalability. (#3652)
Co-authored-by: ShenAo1111 <1377693092@qq.com> Co-authored-by: zhaochenyang20 <zhaochen20@outlook.com>
This commit is contained in:
aoshen524
294
test/srt/models/lora/test_lora.py
Normal file
294
test/srt/models/lora/test_lora.py
Normal file
@@ -0,0 +1,294 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import multiprocessing as mp
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.test.runners import HFRunner, SRTRunner
|
||||
|
||||
LORA_SETS = [
|
||||
# {
|
||||
# "base": "meta-llama/Llama-2-7b-hf",
|
||||
# "loras": ["RuterNorway/Llama-2-7b-chat-norwegian-LoRa"],
|
||||
# },
|
||||
{"base": "meta-llama/Llama-2-7b-hf", "loras": ["winddude/wizardLM-LlaMA-LoRA-7B"]},
|
||||
# {"base": "Qwen/Qwen2.5-14B-Instruct", "loras": ["mssongit/Qwen2.5-14B-SFT-LoRA"]},
|
||||
# {"base": "mistralai/Mistral-7B-Instruct-v0.3", "loras": ["/home/ying/test_lora"]},
|
||||
# {
|
||||
# "base": "mistralai/Mistral-7B-Instruct-v0.3",
|
||||
# "loras": [
|
||||
# "/home/ying/test_lora",
|
||||
# "/home/ying/test_lora_1",
|
||||
# "/home/ying/test_lora_2",
|
||||
# "/home/ying/test_lora_3",
|
||||
# "/home/ying/test_lora_4",
|
||||
# ],
|
||||
# },
|
||||
# {"base": "meta-llama/Llama-2-7b-hf", "loras": ["yard1/llama-2-7b-sql-lora-test"]},
|
||||
]
|
||||
TORCH_DTYPES = [torch.float16]
|
||||
|
||||
PROMPTS = [
|
||||
"""
|
||||
### Instruction:
|
||||
Write a poem about the transformers Python library.
|
||||
Mention the word "large language models" in that poem.
|
||||
### Response:
|
||||
The Transformers are large language models,
|
||||
They're used to make predictions on text.
|
||||
""",
|
||||
"""
|
||||
### Instruction:
|
||||
Tell me about llamas and alpacas
|
||||
### Response:
|
||||
Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids (camels, dromedaries). Llamas live in the Andean mountains of South America where they graze on grasses and shrubs. Alpaca is another name for domesticated llama. The word "alpaca" comes from an Incan language meaning "golden fleece." Alpacas look very similar to llamas but are smaller than their wild relatives. Both species were used by ancient people as pack animals and for meat. Today both llamas and alpacas are raised primarily for their fiber which can be spun into yarn or knitted into clothing.
|
||||
### Question 2:
|
||||
What do you know about llamas?
|
||||
### Answer:
|
||||
""",
|
||||
]
|
||||
|
||||
# import json
|
||||
#
|
||||
# with open("/home/ying/test_prompt/dialogue_choice_prompts.json", "r") as f:
|
||||
# samples = json.load(f)
|
||||
# for sample in samples[:5]:
|
||||
# assert sample[0]["role"] == "user"
|
||||
# PROMPTS.append(sample[0]["content"][:2000])
|
||||
|
||||
|
||||
class TestLoRA(unittest.TestCase):
|
||||
|
||||
def inference(self, prompts, lora_set, tp_size, torch_dtype, max_new_tokens):
|
||||
print("=================== testing inference =======================")
|
||||
base_path = lora_set["base"]
|
||||
all_lora_paths = lora_set["loras"]
|
||||
batch_lora_paths = [None]
|
||||
i = 0
|
||||
for _ in range(len(prompts) - 1):
|
||||
batch_lora_paths.append(all_lora_paths[i])
|
||||
i = (i + 1) % len(all_lora_paths)
|
||||
|
||||
with SRTRunner(
|
||||
base_path,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
tp_size=tp_size,
|
||||
lora_paths=all_lora_paths,
|
||||
max_loras_per_batch=3,
|
||||
disable_cuda_graph=True,
|
||||
disable_radix_cache=True,
|
||||
) as srt_runner:
|
||||
srt_outputs = srt_runner.forward(
|
||||
prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths
|
||||
)
|
||||
|
||||
with HFRunner(
|
||||
base_path, torch_dtype=torch_dtype, model_type="generation"
|
||||
) as hf_runner:
|
||||
hf_outputs = hf_runner.forward(
|
||||
prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths
|
||||
)
|
||||
|
||||
with HFRunner(
|
||||
base_path,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
) as hf_runner:
|
||||
hf_no_lora_outputs = hf_runner.forward(
|
||||
prompts, max_new_tokens=max_new_tokens
|
||||
)
|
||||
|
||||
with SRTRunner(
|
||||
base_path,
|
||||
tp_size=tp_size,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
) as srt_runner:
|
||||
srt_no_lora_outputs = srt_runner.forward(
|
||||
prompts, max_new_tokens=max_new_tokens
|
||||
)
|
||||
|
||||
for i in range(len(prompts)):
|
||||
# compare input logprobs
|
||||
hf_logprobs = torch.Tensor(hf_outputs.top_input_logprobs[i])
|
||||
srt_logprobs = torch.Tensor(srt_outputs.top_input_logprobs[i])
|
||||
hf_no_lora_logprobs = torch.Tensor(hf_no_lora_outputs.top_input_logprobs[i])
|
||||
srt_no_lora_logprobs = torch.Tensor(
|
||||
srt_no_lora_outputs.top_input_logprobs[i]
|
||||
)
|
||||
print(
|
||||
"max input diff between hf_lora and srt_lora",
|
||||
torch.max(abs(hf_logprobs - srt_logprobs)),
|
||||
)
|
||||
print(
|
||||
"max input diff between srt_base and srt_lora",
|
||||
torch.max(abs(srt_no_lora_logprobs - srt_logprobs)),
|
||||
)
|
||||
print(
|
||||
"max input diff between srt_base and hf_base",
|
||||
torch.max(abs(srt_no_lora_logprobs - hf_no_lora_logprobs)),
|
||||
)
|
||||
print(
|
||||
"max input diff between hf_lora and hf_base",
|
||||
torch.max(abs(hf_logprobs - hf_no_lora_logprobs)),
|
||||
)
|
||||
|
||||
# compare output logprobs
|
||||
hf_logprobs = torch.Tensor(hf_outputs.top_output_logprobs[i])
|
||||
srt_logprobs = torch.Tensor(srt_outputs.top_output_logprobs[i])
|
||||
# print(
|
||||
# "\noutput logprobs diff",
|
||||
# [
|
||||
# float(torch.max(abs(hf_logprobs[j] - srt_logprobs[j])))
|
||||
# for j in range(max_new_tokens)
|
||||
# ],
|
||||
# )
|
||||
print(
|
||||
"max output diff between hf_lora and srt_lora",
|
||||
torch.max(abs(hf_logprobs - srt_logprobs)),
|
||||
"\n",
|
||||
)
|
||||
|
||||
# compare output strings
|
||||
print(f"{hf_outputs.output_strs=}")
|
||||
print(f"{srt_outputs.output_strs=}")
|
||||
print(f"{hf_no_lora_outputs.output_strs=}")
|
||||
print(f"{srt_no_lora_outputs.output_strs=}")
|
||||
for i in range(len(prompts)):
|
||||
assert srt_outputs.output_strs[i].strip(" ") == hf_outputs.output_strs[i], (
|
||||
srt_outputs.output_strs[i].strip(" "),
|
||||
hf_outputs.output_strs[i],
|
||||
)
|
||||
# assert (
|
||||
# srt_no_lora_outputs.output_strs[i].strip(" ")
|
||||
# == hf_no_lora_outputs.output_strs[i]
|
||||
# ), (
|
||||
# srt_no_lora_outputs.output_strs[i].strip(" "),
|
||||
# hf_no_lora_outputs.output_strs[i],
|
||||
# )
|
||||
|
||||
def serving(self, prompts, lora_set, tp_size, torch_dtype, max_new_tokens):
|
||||
print("=================== testing serving =======================")
|
||||
# test batch forward
|
||||
base_path = lora_set["base"]
|
||||
all_lora_paths = lora_set["loras"]
|
||||
batch_lora_paths = [None]
|
||||
i = 0
|
||||
for _ in range(len(prompts) - 1):
|
||||
batch_lora_paths.append(all_lora_paths[i])
|
||||
i = (i + 1) % len(all_lora_paths)
|
||||
|
||||
with SRTRunner(
|
||||
base_path,
|
||||
tp_size=tp_size,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
lora_paths=all_lora_paths,
|
||||
max_loras_per_batch=3,
|
||||
disable_cuda_graph=True,
|
||||
disable_radix_cache=True,
|
||||
) as srt_runner:
|
||||
srt_outputs = srt_runner.batch_forward(
|
||||
prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths
|
||||
)
|
||||
|
||||
with HFRunner(
|
||||
base_path,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
output_str_only=True,
|
||||
) as hf_runner:
|
||||
hf_outputs = hf_runner.forward(
|
||||
prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths
|
||||
)
|
||||
|
||||
# compare output strings
|
||||
print(f"{hf_outputs.output_strs=}")
|
||||
print(f"{srt_outputs.output_strs=}")
|
||||
for i in range(len(prompts)):
|
||||
assert srt_outputs.output_strs[i].strip(" ") == hf_outputs.output_strs[i], (
|
||||
srt_outputs.output_strs[i].strip(" "),
|
||||
hf_outputs.output_strs[i],
|
||||
)
|
||||
|
||||
def base_inference(self, prompts, lora_set, tp_size, torch_dtype, max_new_tokens):
|
||||
print("=================== testing base inference =======================")
|
||||
base_path = lora_set["base"]
|
||||
all_lora_paths = lora_set["loras"]
|
||||
batch_lora_paths = [None] * len(prompts)
|
||||
|
||||
with SRTRunner(
|
||||
base_path,
|
||||
tp_size=tp_size,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
) as srt_runner:
|
||||
srt_no_lora_outputs = srt_runner.forward(
|
||||
prompts, max_new_tokens=max_new_tokens
|
||||
)
|
||||
|
||||
with SRTRunner(
|
||||
base_path,
|
||||
tp_size=tp_size,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
lora_paths=all_lora_paths,
|
||||
) as srt_runner:
|
||||
srt_outputs = srt_runner.forward(
|
||||
prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths
|
||||
)
|
||||
|
||||
for i in range(len(prompts)):
|
||||
srt_no_lora_logprobs = torch.Tensor(
|
||||
srt_no_lora_outputs.top_input_logprobs[i]
|
||||
)
|
||||
srt_logprobs = torch.uensor(srt_outputs.top_input_logprobs[i])
|
||||
print("max_diff", torch.max(abs(srt_no_lora_logprobs - srt_logprobs)))
|
||||
|
||||
print(f"{srt_no_lora_outputs.output_strs=}")
|
||||
print(f"{srt_outputs.output_strs=}")
|
||||
|
||||
for i in range(len(prompts)):
|
||||
assert srt_outputs.output_strs[i].strip(" ") == hf_outputs.output_strs[i], (
|
||||
srt_outputs.output_strs[i].strip(" "),
|
||||
hf_outputs.output_strs[i],
|
||||
)
|
||||
assert (
|
||||
srt_no_lora_outputs[i].output_strs.strip(" ")
|
||||
== hf_no_lora_outputs[i].output_strs
|
||||
)
|
||||
|
||||
def test_all(self):
|
||||
for lora_set in LORA_SETS:
|
||||
# self.load_lora_adapter(lora_set, 1)
|
||||
for torch_dtype in TORCH_DTYPES:
|
||||
tp_size = 1
|
||||
max_new_tokens = 32
|
||||
self.inference(PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens)
|
||||
# self.serving(PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens)
|
||||
# self.base_inference(
|
||||
# PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens
|
||||
# )
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
mp.set_start_method("spawn")
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
unittest.main(warnings="ignore")
|
||||
253
test/srt/models/lora/test_lora_backend.py
Normal file
253
test/srt/models/lora/test_lora_backend.py
Normal file
@@ -0,0 +1,253 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
from utils import *
|
||||
|
||||
from sglang.test.runners import HFRunner, SRTRunner
|
||||
from sglang.test.test_utils import calculate_rouge_l, is_in_ci
|
||||
|
||||
CI_LORA_MODELS = [
|
||||
LoRAModelCase(
|
||||
base="meta-llama/Llama-3.1-8B-Instruct",
|
||||
adaptors=[
|
||||
LoRAAdaptor(
|
||||
name="algoprog/fact-generation-llama-3.1-8b-instruct-lora",
|
||||
),
|
||||
],
|
||||
max_loras_per_batch=1,
|
||||
),
|
||||
LoRAModelCase(
|
||||
base="meta-llama/Llama-3.1-8B-Instruct",
|
||||
adaptors=[
|
||||
LoRAAdaptor(
|
||||
name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
|
||||
prefill_tolerance=1e-1,
|
||||
),
|
||||
],
|
||||
max_loras_per_batch=1,
|
||||
),
|
||||
]
|
||||
|
||||
ALL_OTHER_LORA_MODELS = [
|
||||
LoRAModelCase(
|
||||
base="meta-llama/Llama-2-7b-hf",
|
||||
adaptors=[LoRAAdaptor(name="winddude/wizardLM-LlaMA-LoRA-7B")],
|
||||
max_loras_per_batch=1,
|
||||
),
|
||||
]
|
||||
|
||||
PROMPTS = [
|
||||
"AI is a field of computer science focused on",
|
||||
"""
|
||||
### Instruction:
|
||||
Tell me about llamas and alpacas
|
||||
### Response:
|
||||
Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids (camels, dromedaries). Llamas live in the Andean mountains of South America where they graze on grasses and shrubs. Alpaca is another name for domesticated llama. The word "alpaca" comes from an Incan language meaning "golden fleece." Alpacas look very similar to llamas but are smaller than their wild relatives. Both species were used by ancient people as pack animals and for meat. Today both llamas and alpacas are raised primarily for their fiber which can be spun into yarn or knitted into clothing.
|
||||
### Question 2:
|
||||
What do you know about llamas?
|
||||
### Answer:
|
||||
""",
|
||||
]
|
||||
|
||||
|
||||
class TestLoRABackend(unittest.TestCase):
|
||||
def run_backend(
|
||||
self,
|
||||
prompt: str,
|
||||
model_case: LoRAModelCase,
|
||||
torch_dtype: torch.dtype,
|
||||
max_new_tokens: int,
|
||||
backend: str,
|
||||
):
|
||||
"""
|
||||
Run backend tests for a single prompt and model case.
|
||||
"""
|
||||
base_path = model_case.base
|
||||
adaptor = model_case.adaptors[0]
|
||||
print(
|
||||
f"\n========== Testing backend '{backend}' for base '{base_path}' --- "
|
||||
f"Prompt '{prompt[:50]}...' using adaptor '{adaptor.name}' ---"
|
||||
)
|
||||
with SRTRunner(
|
||||
base_path,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
tp_size=model_case.tp_size,
|
||||
lora_paths=[adaptor.name for adaptor in model_case.adaptors],
|
||||
max_loras_per_batch=model_case.max_loras_per_batch,
|
||||
lora_backend=backend,
|
||||
disable_cuda_graph=True,
|
||||
disable_radix_cache=True,
|
||||
mem_fraction_static=0.88,
|
||||
) as srt_runner:
|
||||
srt_outputs = srt_runner.forward(
|
||||
[prompt], max_new_tokens=max_new_tokens, lora_paths=[adaptor.name]
|
||||
)
|
||||
|
||||
with HFRunner(
|
||||
base_path, torch_dtype=torch_dtype, model_type="generation"
|
||||
) as hf_runner:
|
||||
hf_outputs = hf_runner.forward(
|
||||
[prompt], max_new_tokens=max_new_tokens, lora_paths=[adaptor.name]
|
||||
)
|
||||
|
||||
with SRTRunner(
|
||||
base_path,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
tp_size=model_case.tp_size,
|
||||
mem_fraction_static=0.88,
|
||||
) as srt_runner:
|
||||
srt_no_lora_outputs = srt_runner.forward(
|
||||
[prompt], max_new_tokens=max_new_tokens
|
||||
)
|
||||
|
||||
with HFRunner(
|
||||
base_path,
|
||||
torch_dtype=torch_dtype,
|
||||
model_type="generation",
|
||||
) as hf_runner:
|
||||
hf_no_lora_outputs = hf_runner.forward(
|
||||
[prompt], max_new_tokens=max_new_tokens
|
||||
)
|
||||
|
||||
# Use individual adapter tolerances if set, otherwise use model defaults
|
||||
prefill_tol = (
|
||||
adaptor.prefill_tolerance
|
||||
if adaptor.prefill_tolerance is not None
|
||||
else model_case.prefill_tolerance
|
||||
)
|
||||
decode_tol = (
|
||||
adaptor.decode_tolerance
|
||||
if adaptor.decode_tolerance is not None
|
||||
else model_case.decode_tolerance
|
||||
)
|
||||
rouge_tol = (
|
||||
adaptor.rouge_l_tolerance
|
||||
if adaptor.rouge_l_tolerance is not None
|
||||
else model_case.rouge_l_tolerance
|
||||
)
|
||||
|
||||
# Compare prefill stage logprobs (HF vs SRTRunner with LoRA)
|
||||
hf_prefill = torch.tensor(hf_outputs.top_input_logprobs[0])
|
||||
srt_prefill = torch.tensor(srt_outputs.top_input_logprobs[0])
|
||||
max_prefill_diff = torch.max(torch.abs(hf_prefill - srt_prefill))
|
||||
print("Max prefill diff (HF vs SRT):", max_prefill_diff)
|
||||
|
||||
# Compare decode stage logprobs
|
||||
hf_decode = torch.tensor(hf_outputs.top_output_logprobs[0])
|
||||
srt_decode = torch.tensor(srt_outputs.top_output_logprobs[0])
|
||||
max_decode_diff = torch.max(torch.abs(hf_decode - srt_decode))
|
||||
print("Max decode diff (HF vs SRT):", max_decode_diff)
|
||||
|
||||
srt_output_str = srt_outputs.output_strs[0].strip()
|
||||
hf_output_str = hf_outputs.output_strs[0].strip()
|
||||
rouge_score = calculate_rouge_l([srt_output_str], [hf_output_str])[0]
|
||||
print("ROUGE-L score:", rouge_score)
|
||||
print("SRT output:", srt_output_str)
|
||||
print("HF output:", hf_output_str)
|
||||
|
||||
# Additional: compare prefill outputs between base model (no LoRA) and LoRA model for reference
|
||||
hf_no_lora_prefill = torch.tensor(hf_no_lora_outputs.top_input_logprobs[0])
|
||||
srt_no_lora_prefill = torch.tensor(srt_no_lora_outputs.top_input_logprobs[0])
|
||||
print(
|
||||
"Max diff (SRT base vs SRT LoRA prefill):",
|
||||
torch.max(torch.abs(srt_no_lora_prefill - srt_prefill)),
|
||||
)
|
||||
print(
|
||||
"Max diff (HF base vs HF LoRA prefill):",
|
||||
torch.max(torch.abs(hf_no_lora_prefill - hf_prefill)),
|
||||
)
|
||||
|
||||
if hf_prefill.shape[0] <= 100:
|
||||
assert torch.all(torch.abs(hf_prefill - srt_prefill) < prefill_tol), (
|
||||
f"Prefill logprobs mismatch for base '{base_path}', adaptor '{adaptor.name}', "
|
||||
f"backend '{backend}', prompt: '{prompt[:50]}...'"
|
||||
)
|
||||
|
||||
if hf_decode.shape[0] <= 100:
|
||||
assert torch.all(torch.abs(hf_decode - srt_decode) < decode_tol), (
|
||||
f"Decode logprobs mismatch for base '{base_path}', adaptor '{adaptor.name}', "
|
||||
f"backend '{backend}', prompt: '{prompt[:50]}...'"
|
||||
)
|
||||
|
||||
if rouge_score < rouge_tol:
|
||||
|
||||
raise AssertionError(
|
||||
f"ROUGE-L score {rouge_score} below tolerance {rouge_tol} "
|
||||
f"for base '{base_path}', adaptor '{adaptor.name}', backend '{backend}', prompt: '{prompt[:50]}...'"
|
||||
)
|
||||
|
||||
def run_backend_batch(
|
||||
self,
|
||||
prompts: List[str],
|
||||
model_case: LoRAModelCase,
|
||||
torch_dtype: torch.dtype,
|
||||
max_new_tokens: int,
|
||||
backend: str,
|
||||
):
|
||||
# TODO: Implement batch processing version of run_backend
|
||||
raise NotImplementedError(
|
||||
"Batch processing version of run_backend is not implemented yet."
|
||||
)
|
||||
|
||||
def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]):
|
||||
for model_case in model_cases:
|
||||
# If skip_long_prompt is True, filter out prompts longer than 1000 characters
|
||||
prompts = (
|
||||
PROMPTS
|
||||
if not model_case.skip_long_prompt
|
||||
else [p for p in PROMPTS if len(p) < 1000]
|
||||
)
|
||||
for torch_dtype in TORCH_DTYPES:
|
||||
for backend in BACKENDS:
|
||||
for prompt in prompts:
|
||||
self.run_backend(
|
||||
prompt,
|
||||
model_case,
|
||||
torch_dtype,
|
||||
max_new_tokens=32,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
def test_ci_lora_models(self):
|
||||
self._run_backend_on_model_cases(CI_LORA_MODELS)
|
||||
|
||||
def test_all_lora_models(self):
|
||||
if is_in_ci():
|
||||
return
|
||||
|
||||
# Retain ONLY_RUN check here
|
||||
filtered_models = []
|
||||
for model_case in ALL_OTHER_LORA_MODELS:
|
||||
if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base:
|
||||
continue
|
||||
filtered_models.append(model_case)
|
||||
|
||||
self._run_backend_on_model_cases(filtered_models)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
mp.set_start_method("spawn")
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
unittest.main(warnings="ignore")
|
||||
109
test/srt/models/lora/test_multi_lora_backend.py
Normal file
109
test/srt/models/lora/test_multi_lora_backend.py
Normal file
@@ -0,0 +1,109 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
from utils import *
|
||||
|
||||
from sglang.test.runners import HFRunner, SRTRunner
|
||||
from sglang.test.test_utils import calculate_rouge_l, is_in_ci
|
||||
|
||||
MULTI_LORA_MODELS = [
|
||||
LoRAModelCase(
|
||||
base="meta-llama/Llama-3.1-8B-Instruct",
|
||||
adaptors=[
|
||||
LoRAAdaptor(
|
||||
name="algoprog/fact-generation-llama-3.1-8b-instruct-lora",
|
||||
),
|
||||
LoRAAdaptor(
|
||||
name="some-org/another-lora-adaptor",
|
||||
),
|
||||
],
|
||||
max_loras_per_batch=2,
|
||||
),
|
||||
]
|
||||
|
||||
# All prompts are used at once in a batch.
|
||||
PROMPTS = [
|
||||
"AI is a field of computer science focused on",
|
||||
"""
|
||||
### Instruction:
|
||||
Tell me about llamas and alpacas
|
||||
### Response:
|
||||
Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids.
|
||||
### Question:
|
||||
What do you know about llamas?
|
||||
### Answer:
|
||||
""",
|
||||
]
|
||||
|
||||
|
||||
class TestMultiLoRABackend(unittest.TestCase):
|
||||
def run_backend_batch(
|
||||
self,
|
||||
prompts: List[str],
|
||||
model_case: LoRAModelCase,
|
||||
torch_dtype: torch.dtype,
|
||||
max_new_tokens: int,
|
||||
backend: str,
|
||||
):
|
||||
"""
|
||||
The multi-LoRA backend test functionality is not supported yet.
|
||||
This function uses all prompts at once and prints a message indicating that support is pending.
|
||||
"""
|
||||
adaptor_names = [adaptor.name for adaptor in model_case.adaptors]
|
||||
print(
|
||||
f"\n========== Testing multi-LoRA backend '{backend}' for base '{model_case.base}' --- "
|
||||
f"Using prompts {[p[:50] for p in prompts]} with adaptors: {adaptor_names} ---"
|
||||
)
|
||||
print(
|
||||
"run_backend_batch: Multi-LoRA backend test functionality is pending support."
|
||||
)
|
||||
|
||||
def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]):
|
||||
for model_case in model_cases:
|
||||
# If skip_long_prompt is True, filter out prompts longer than 1000 characters.
|
||||
batch_prompts = (
|
||||
PROMPTS
|
||||
if not model_case.skip_long_prompt
|
||||
else [p for p in PROMPTS if len(p) < 1000]
|
||||
)
|
||||
for torch_dtype in TORCH_DTYPES:
|
||||
for backend in BACKENDS:
|
||||
self.run_backend_batch(
|
||||
batch_prompts,
|
||||
model_case,
|
||||
torch_dtype,
|
||||
max_new_tokens=32,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
def test_multi_lora_models(self):
|
||||
# Optionally skip tests in CI environments.
|
||||
if is_in_ci():
|
||||
return
|
||||
self._run_backend_on_model_cases(MULTI_LORA_MODELS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
mp.set_start_method("spawn")
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
unittest.main(warnings="ignore")
|
||||
49
test/srt/models/lora/utils.py
Normal file
49
test/srt/models/lora/utils.py
Normal file
@@ -0,0 +1,49 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import dataclasses
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class LoRAAdaptor:
|
||||
name: str
|
||||
prefill_tolerance: float = None
|
||||
decode_tolerance: float = None
|
||||
rouge_l_tolerance: float = None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class LoRAModelCase:
|
||||
base: str
|
||||
adaptors: List[LoRAAdaptor]
|
||||
tp_size: int = 1
|
||||
prefill_tolerance: float = 5e-2
|
||||
decode_tolerance: float = 5e-2
|
||||
rouge_l_tolerance: float = 1.0
|
||||
max_loras_per_batch: int = 1
|
||||
skip_long_prompt: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
if len(self.adaptors) > self.max_loras_per_batch:
|
||||
raise ValueError(
|
||||
f"For base '{self.base}', number of adaptors ({len(self.adaptors)}) "
|
||||
f"must be <= max_loras_per_batch ({self.max_loras_per_batch})"
|
||||
)
|
||||
|
||||
|
||||
TORCH_DTYPES = [torch.float16]
|
||||
BACKENDS = ["triton"]
|
||||
Reference in New Issue
Block a user