EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Crash the server correctly during error (#2231)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-11-28 00:22:39 -08:00
committed by GitHub
parent db674e3d24
commit d4fc1a70e3
46 changed files with 147 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
test/srt/sampling/penaltylib/test_srt_endpoint_with_penalizers.py
View File

@@ -4,7 +4,7 @@ from multiprocessing import Process
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -31,7 +31,7 @@ class TestBatchPenalizerE2E(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def run_decode(
self,

4
test/srt/test_cache_report.py
View File

@@ -4,7 +4,7 @@ import unittest
import openai
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_URL_FOR_TEST,
@@ -44,7 +44,7 @@ class TestCacheReport(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def run_decode(self, return_logprob=False, top_logprobs_num=0, n=1):
response = requests.post(

4
test/srt/test_data_parallelism.py
View File

@@ -4,7 +4,7 @@ from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -28,7 +28,7 @@ class TestDataParallelism(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_double_sparsity.py
View File

@@ -2,7 +2,7 @@ import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -45,7 +45,7 @@ class TestDoubleSparsity(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_dp_attention.py
View File

@@ -1,7 +1,7 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
@@ -30,7 +30,7 @@ class TestDPAttention(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_embedding_openai_server.py
View File

@@ -3,7 +3,7 @@ import unittest
import openai
from sglang.srt.hf_transformers_utils import get_tokenizer
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -28,7 +28,7 @@ class TestOpenAIServer(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def run_embedding(self, use_list_input, token_input):
client = openai.Client(api_key=self.api_key, base_url=self.base_url)

4
test/srt/test_eval_accuracy_large.py
View File

@@ -6,7 +6,7 @@ python -m unittest test_eval_accuracy_large.TestEvalAccuracyLarge.test_mmlu
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -30,7 +30,7 @@ class TestEvalAccuracyLarge(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_eval_accuracy_large_chunked_prefill.py
View File

@@ -1,7 +1,7 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -25,7 +25,7 @@ class TestEvalAccuracyLargeChunkedPrefill(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_eval_accuracy_large_mixed_chunked_prefill.py
View File

@@ -1,7 +1,7 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -31,7 +31,7 @@ class TestEvalAccuracyLargeChunkedPrefill(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_eval_accuracy_mini.py
View File

@@ -1,7 +1,7 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -22,7 +22,7 @@ class TestEvalAccuracyMini(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_input_embeddings.py
View File

@@ -4,7 +4,7 @@ import unittest
import requests
from transformers import AutoModelForCausalLM, AutoTokenizer
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -107,7 +107,7 @@ class TestInputEmbeds(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
if __name__ == "__main__":

4
test/srt/test_json_constrained.py
View File

@@ -9,7 +9,7 @@ from concurrent.futures import ThreadPoolExecutor
import openai
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_URL_FOR_TEST,
@@ -46,7 +46,7 @@ class TestJSONConstrainedOutlinesBackend(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def run_decode(self, json_schema, return_logprob=False, top_logprobs_num=0, n=1):
response = requests.post(

4
test/srt/test_large_max_new_tokens.py
View File

@@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor
import openai
from sglang.srt.hf_transformers_utils import get_tokenizer
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -52,7 +52,7 @@ class TestLargeMaxNewTokens(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
cls.stdout.close()
cls.stderr.close()
os.remove(STDOUT_FILENAME)

4
test/srt/test_matched_stop.py
View File

@@ -3,7 +3,7 @@ import unittest
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_URL_FOR_TEST,
@@ -32,7 +32,7 @@ class TestMatchedStop(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def run_completions_generation(
self,

4
test/srt/test_metrics.py
View File

@@ -2,7 +2,7 @@ import unittest
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -75,7 +75,7 @@ class TestEnableMetrics(unittest.TestCase):
self.assertIn("_bucket{", metrics_content)
finally:
kill_child_process(process.pid, include_self=True)
kill_process_tree(process.pid)
if __name__ == "__main__":

4
test/srt/test_mla.py
View File

@@ -1,7 +1,7 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
@@ -25,7 +25,7 @@ class TestMLA(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_mla_fp8.py
View File

@@ -1,7 +1,7 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST,
@@ -31,7 +31,7 @@ class TestMLA(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mgsm_en(self):
args = SimpleNamespace(

4
test/srt/test_moe_eval_accuracy_large.py
View File

@@ -6,7 +6,7 @@ python -m unittest test_moe_eval_accuracy_large.TestMoEEvalAccuracyLarge.test_mm
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
@@ -35,7 +35,7 @@ class TestMoEEvalAccuracyLarge(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_nightly_gsm8k_eval.py
View File

@@ -6,7 +6,7 @@ import warnings
from datetime import datetime
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
@@ -132,7 +132,7 @@ class TestEvalAccuracyLarge(unittest.TestCase):
def tearDown(self):
if self.process:
kill_child_process(self.process.pid, include_self=True)
kill_process_tree(self.process.pid)
def test_mgsm_en_all_models(self):
warnings.filterwarnings(

6
test/srt/test_nightly_human_eval.py
View File

@@ -6,7 +6,7 @@ import unittest
from test_nightly_gsm8k_eval import launch_server, parse_models
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
@@ -32,9 +32,9 @@ class TestEvalAccuracyLarge(unittest.TestCase):
@classmethod
def tearDownClass(cls):
if cls.process:
kill_child_process(cls.process.pid)
kill_process_tree(cls.process.pid)
if cls.eval_process:
kill_child_process(cls.eval_process.pid)
kill_process_tree(cls.eval_process.pid)
def run_evalplus(self, model):
print("Delete evalplus results")

4
test/srt/test_openai_server.py
View File

@@ -11,7 +11,7 @@ import unittest
import openai
from sglang.srt.hf_transformers_utils import get_tokenizer
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -37,7 +37,7 @@ class TestOpenAIServer(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def run_completion(
self, echo, logprobs, use_list_input, parallel_sample_num, token_input

4
test/srt/test_pytorch_sampling_backend.py
View File

@@ -3,7 +3,7 @@ from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -27,7 +27,7 @@ class TestPyTorchSamplingBackend(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_radix_attention.py
View File

@@ -8,7 +8,7 @@ from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
kill_child_process,
kill_process_tree,
popen_launch_server,
)
@@ -80,7 +80,7 @@ class TestRadixCacheFCFS(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_radix_attention(self):
nodes = gen_radix_tree()

4
test/srt/test_retract_decode.py
View File

@@ -1,7 +1,7 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -22,7 +22,7 @@ class TestRetractDecode(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

6
test/srt/test_session_control.py
View File

@@ -9,7 +9,7 @@ import unittest
import requests
from sglang.srt.hf_transformers_utils import get_tokenizer
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -29,7 +29,7 @@ class TestSessionControl(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_session_control(self):
chunks = [
@@ -191,7 +191,7 @@ class TestSessionControlVision(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_session_control(self):
text_chunks = [

4
test/srt/test_skip_tokenizer_init.py
View File

@@ -7,7 +7,7 @@ import unittest
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -30,7 +30,7 @@ class TestSkipTokenizerInit(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def run_decode(self, return_logprob=False, top_logprobs_num=0, n=1):
max_new_tokens = 32

4
test/srt/test_srt_endpoint.py
View File

@@ -9,7 +9,7 @@ import unittest
import numpy as np
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -29,7 +29,7 @@ class TestSRTEndpoint(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def run_decode(
self,

4
test/srt/test_torch_compile.py
View File

@@ -4,7 +4,7 @@ from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -28,7 +28,7 @@ class TestTorchCompile(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_torch_compile_moe.py
View File

@@ -4,7 +4,7 @@ from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST,
@@ -28,7 +28,7 @@ class TestTorchCompile(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_torchao.py
View File

@@ -3,7 +3,7 @@ from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -27,7 +27,7 @@ class TestTorchAO(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(

4
test/srt/test_triton_attention_backend.py
View File

@@ -6,7 +6,7 @@ python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mml
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -54,7 +54,7 @@ class TestTritonAttnBackend(unittest.TestCase):
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
finally:
kill_child_process(process.pid, include_self=True)
kill_process_tree(process.pid)
if __name__ == "__main__":

4
test/srt/test_update_weights.py
View File

@@ -3,7 +3,7 @@ import unittest
import requests
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -23,7 +23,7 @@ class TestUpdateWeights(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def run_decode(self):
response = requests.post(

6
test/srt/test_vision_openai_server.py
View File

@@ -17,7 +17,7 @@ import requests
from decord import VideoReader, cpu
from PIL import Image
from sglang.srt.utils import kill_child_process
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -46,7 +46,7 @@ class TestOpenAIVisionServer(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_chat_completion(self):
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
@@ -387,7 +387,7 @@ class TestQWen2VLServerContextLengthIssue(unittest.TestCase):
@classmethod
def tearDownClass(cls):
kill_child_process(cls.process.pid, include_self=True)
kill_process_tree(cls.process.pid)
def test_chat_completion(self):
client = openai.Client(api_key=self.api_key, base_url=self.base_url)