EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

misc: add pre-commit config (#637)

Browse Source
This commit is contained in:
zhyncs
2024-07-18 04:55:39 +10:00
committed by GitHub
parent a8552cb18b
commit 2e341cd493
43 changed files with 481 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
examples/quick_start/anthropic_example_chat.py
View File

@@ -3,6 +3,7 @@ Usage:
export ANTHROPIC_API_KEY=sk-******
python3 anthropic_example_chat.py
"""
import sglang as sgl
@@ -30,7 +31,7 @@ def stream():
state = multi_turn_question.run(
question_1="What is the capital of the United States?",
question_2="List two local attractions.",
stream=True
stream=True,
)
for out in state.text_iter():
@@ -39,13 +40,18 @@ def stream():
def batch():
states = multi_turn_question.run_batch([
{"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions."},
{"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?"},
])
states = multi_turn_question.run_batch(
[
{
"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions.",
},
{
"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?",
},
]
)
for s in states:
print(s.messages())

19
examples/quick_start/anthropic_example_complete.py
View File

@@ -9,15 +9,14 @@ import sglang as sgl
@sgl.function
def few_shot_qa(s, question):
s += (
"""
s += """
\n\nHuman: What is the capital of France?
\n\nAssistant: Paris
\n\nHuman: What is the capital of Germany?
\n\nAssistant: Berlin
\n\nHuman: What is the capital of Italy?
\n\nAssistant: Rome
""")
"""
s += "\n\nHuman: " + question + "\n"
s += "\n\nAssistant:" + sgl.gen("answer", temperature=0)
@@ -33,8 +32,8 @@ def single():
def stream():
state = few_shot_qa.run(
question="What is the capital of the United States?",
stream=True)
question="What is the capital of the United States?", stream=True
)
for out in state.text_iter("answer"):
print(out, end="", flush=True)
@@ -42,10 +41,12 @@ def stream():
def batch():
states = few_shot_qa.run_batch([
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
])
states = few_shot_qa.run_batch(
[
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
]
)
for s in states:
print(s["answer"])

25
examples/quick_start/azure_openai_example_chat.py
View File

@@ -3,9 +3,11 @@ Usage:
export AZURE_OPENAI_API_KEY=sk-******
python3 openai_example_chat.py
"""
import sglang as sgl
import os
import sglang as sgl
@sgl.function
def multi_turn_question(s, question_1, question_2):
@@ -32,7 +34,7 @@ def stream():
state = multi_turn_question.run(
question_1="What is the capital of the United States?",
question_2="List two local attractions.",
stream=True
stream=True,
)
for out in state.text_iter():
@@ -41,13 +43,18 @@ def stream():
def batch():
states = multi_turn_question.run_batch([
{"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions."},
{"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?"},
])
states = multi_turn_question.run_batch(
[
{
"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions.",
},
{
"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?",
},
]
)
for s in states:
print(s.messages())

22
examples/quick_start/gemini_example_chat.py
View File

@@ -3,6 +3,7 @@ Usage:
export GCP_PROJECT_ID=******
python3 gemini_example_chat.py
"""
import sglang as sgl
@@ -30,7 +31,7 @@ def stream():
state = multi_turn_question.run(
question_1="What is the capital of the United States?",
question_2="List two local attractions.",
stream=True
stream=True,
)
for out in state.text_iter():
@@ -39,13 +40,18 @@ def stream():
def batch():
states = multi_turn_question.run_batch([
{"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions."},
{"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?"},
])
states = multi_turn_question.run_batch(
[
{
"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions.",
},
{
"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?",
},
]
)
for s in states:
print(s.messages())

19
examples/quick_start/gemini_example_complete.py
View File

@@ -9,15 +9,14 @@ import sglang as sgl
@sgl.function
def few_shot_qa(s, question):
s += (
"""The following are questions with answers.
s += """The following are questions with answers.
Q: What is the capital of France?
A: Paris
Q: What is the capital of Germany?
A: Berlin
Q: What is the capital of Italy?
A: Rome
""")
"""
s += "Q: " + question + "\n"
s += "A:" + sgl.gen("answer", stop="\n", temperature=0)
@@ -33,8 +32,8 @@ def single():
def stream():
state = few_shot_qa.run(
question="What is the capital of the United States?",
stream=True)
question="What is the capital of the United States?", stream=True
)
for out in state.text_iter("answer"):
print(out, end="", flush=True)
@@ -42,10 +41,12 @@ def stream():
def batch():
states = few_shot_qa.run_batch([
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
])
states = few_shot_qa.run_batch(
[
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
]
)
for s in states:
print(s["answer"])

3
examples/quick_start/gemini_example_multimodal_chat.py
View File

@@ -3,6 +3,7 @@ Usage:
export GCP_PROJECT_ID=******
python3 gemini_example_multimodal_chat.py
"""
import sglang as sgl
@@ -19,7 +20,7 @@ if __name__ == "__main__":
image_file1="./images/cat.jpeg",
image_file2="./images/dog.jpeg",
question="Describe difference of the two images in one sentence.",
stream=True
stream=True,
)
for out in state.text_iter("answer"):

22
examples/quick_start/openai_example_chat.py
View File

@@ -3,6 +3,7 @@ Usage:
export OPENAI_API_KEY=sk-******
python3 openai_example_chat.py
"""
import sglang as sgl
@@ -31,7 +32,7 @@ def stream():
state = multi_turn_question.run(
question_1="What is the capital of the United States?",
question_2="List two local attractions.",
stream=True
stream=True,
)
for out in state.text_iter():
@@ -40,13 +41,18 @@ def stream():
def batch():
states = multi_turn_question.run_batch([
{"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions."},
{"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?"},
])
states = multi_turn_question.run_batch(
[
{
"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions.",
},
{
"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?",
},
]
)
for s in states:
print(s.messages())

19
examples/quick_start/openai_example_complete.py
View File

@@ -9,15 +9,14 @@ import sglang as sgl
@sgl.function
def few_shot_qa(s, question):
s += (
"""The following are questions with answers.
s += """The following are questions with answers.
Q: What is the capital of France?
A: Paris
Q: What is the capital of Germany?
A: Berlin
Q: What is the capital of Italy?
A: Rome
""")
"""
s += "Q: " + question + "\n"
s += "A:" + sgl.gen("answer", stop="\n", temperature=0)
@@ -33,8 +32,8 @@ def single():
def stream():
state = few_shot_qa.run(
question="What is the capital of the United States?",
stream=True)
question="What is the capital of the United States?", stream=True
)
for out in state.text_iter("answer"):
print(out, end="", flush=True)
@@ -42,10 +41,12 @@ def stream():
def batch():
states = few_shot_qa.run_batch([
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
])
states = few_shot_qa.run_batch(
[
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
]
)
for s in states:
print(s["answer"])

4
examples/quick_start/openrouter_example_chat.py
View File

@@ -3,9 +3,11 @@ Usage:
export OPENROUTER_API_KEY=sk-******
python3 together_example_chat.py
"""
import sglang as sgl
import os
import sglang as sgl
@sgl.function
def multi_turn_question(s, question_1, question_2):

22
examples/quick_start/srt_example_chat.py
View File

@@ -2,6 +2,7 @@
Usage:
python3 srt_example_chat.py
"""
import sglang as sgl
@@ -29,7 +30,7 @@ def stream():
state = multi_turn_question.run(
question_1="What is the capital of the United States?",
question_2="List two local attractions.",
stream=True
stream=True,
)
for out in state.text_iter():
@@ -38,13 +39,18 @@ def stream():
def batch():
states = multi_turn_question.run_batch([
{"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions."},
{"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?"},
])
states = multi_turn_question.run_batch(
[
{
"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions.",
},
{
"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?",
},
]
)
for s in states:
print(s.messages())

20
examples/quick_start/srt_example_complete.py
View File

@@ -2,20 +2,20 @@
Usage:
python3 srt_example_complete.py
"""
import sglang as sgl
@sgl.function
def few_shot_qa(s, question):
s += (
"""The following are questions with answers.
s += """The following are questions with answers.
Q: What is the capital of France?
A: Paris
Q: What is the capital of Germany?
A: Berlin
Q: What is the capital of Italy?
A: Rome
""")
"""
s += "Q: " + question + "\n"
s += "A:" + sgl.gen("answer", stop="\n", temperature=0)
@@ -31,8 +31,8 @@ def single():
def stream():
state = few_shot_qa.run(
question="What is the capital of the United States?",
stream=True)
question="What is the capital of the United States?", stream=True
)
for out in state.text_iter("answer"):
print(out, end="", flush=True)
@@ -40,10 +40,12 @@ def stream():
def batch():
states = few_shot_qa.run_batch([
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
])
states = few_shot_qa.run_batch(
[
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
]
)
for s in states:
print(s["answer"])

19
examples/quick_start/srt_example_llava.py
View File

@@ -1,6 +1,7 @@
"""
Usage: python3 srt_example_llava.py
"""
import sglang as sgl
@@ -12,9 +13,8 @@ def image_qa(s, image_path, question):
def single():
state = image_qa.run(
image_path="images/cat.jpeg",
question="What is this?",
max_new_tokens=128)
image_path="images/cat.jpeg", question="What is this?", max_new_tokens=128
)
print(state["answer"], "\n")
@@ -23,7 +23,8 @@ def stream():
image_path="images/cat.jpeg",
question="What is this?",
max_new_tokens=64,
stream=True)
stream=True,
)
for out in state.text_iter("answer"):
print(out, end="", flush=True)
@@ -33,8 +34,8 @@ def stream():
def batch():
states = image_qa.run_batch(
[
{"image_path": "images/cat.jpeg", "question":"What is this?"},
{"image_path": "images/dog.jpeg", "question":"What is this?"},
{"image_path": "images/cat.jpeg", "question": "What is this?"},
{"image_path": "images/dog.jpeg", "question": "What is this?"},
],
max_new_tokens=128,
)
@@ -43,8 +44,10 @@ def batch():
if __name__ == "__main__":
runtime = sgl.Runtime(model_path="liuhaotian/llava-v1.6-vicuna-7b",
tokenizer_path="llava-hf/llava-1.5-7b-hf")
runtime = sgl.Runtime(
model_path="liuhaotian/llava-v1.6-vicuna-7b",
tokenizer_path="llava-hf/llava-1.5-7b-hf",
)
sgl.set_default_backend(runtime)
print(f"chat template: {runtime.endpoint.chat_template.name}")

13
examples/quick_start/srt_example_yi_vl.py
View File

@@ -3,6 +3,7 @@ Usage: python3 srt_example_yi_vl.py
Requirements: transformers==4.38
"""
import sglang as sgl
@@ -17,7 +18,8 @@ def single():
image_path="images/cat.jpeg",
question="What is this?",
max_new_tokens=64,
stop="###")
stop="###",
)
print(state["answer"], "\n")
@@ -27,7 +29,8 @@ def stream():
question="What is this?",
max_new_tokens=64,
stream=True,
stop="###")
stop="###",
)
for out in state.text_iter("answer"):
print(out, end="", flush=True)
@@ -37,11 +40,11 @@ def stream():
def batch():
states = image_qa.run_batch(
[
{"image_path": "images/cat.jpeg", "question":"What is this?"},
{"image_path": "images/dog.jpeg", "question":"What is this?"},
{"image_path": "images/cat.jpeg", "question": "What is this?"},
{"image_path": "images/dog.jpeg", "question": "What is this?"},
],
max_new_tokens=64,
stop="###"
stop="###",
)
for s in states:
print(s["answer"], "\n")

25
examples/quick_start/together_example_chat.py
View File

@@ -3,9 +3,11 @@ Usage:
export TOGETHER_API_KEY=sk-******
python3 together_example_chat.py
"""
import sglang as sgl
import os
import sglang as sgl
@sgl.function
def multi_turn_question(s, question_1, question_2):
@@ -32,7 +34,7 @@ def stream():
state = multi_turn_question.run(
question_1="What is the capital of the United States?",
question_2="List two local attractions.",
stream=True
stream=True,
)
for out in state.text_iter():
@@ -41,13 +43,18 @@ def stream():
def batch():
states = multi_turn_question.run_batch([
{"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions."},
{"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?"},
])
states = multi_turn_question.run_batch(
[
{
"question_1": "What is the capital of the United States?",
"question_2": "List two local attractions.",
},
{
"question_1": "What is the capital of France?",
"question_2": "What is the population of this city?",
},
]
)
for s in states:
print(s.messages())

22
examples/quick_start/together_example_complete.py
View File

@@ -4,21 +4,21 @@ export TOGETHER_API_KEY=sk-******
python3 together_example_complete.py
"""
import sglang as sgl
import os
import sglang as sgl
@sgl.function
def few_shot_qa(s, question):
s += (
"""The following are questions with answers.
s += """The following are questions with answers.
Q: What is the capital of France?
A: Paris
Q: What is the capital of Germany?
A: Berlin
Q: What is the capital of Italy?
A: Rome
""")
"""
s += "Q: " + question + "\n"
s += "A:" + sgl.gen("answer", stop="\n", temperature=0)
@@ -34,8 +34,8 @@ def single():
def stream():
state = few_shot_qa.run(
question="What is the capital of the United States?",
stream=True)
question="What is the capital of the United States?", stream=True
)
for out in state.text_iter("answer"):
print(out, end="", flush=True)
@@ -43,10 +43,12 @@ def stream():
def batch():
states = few_shot_qa.run_batch([
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
])
states = few_shot_qa.run_batch(
[
{"question": "What is the capital of the United States?"},
{"question": "What is the capital of China?"},
]
)
for s in states:
print(s["answer"])