EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Improve Readme (#10)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-01-15 21:37:11 -08:00
parent 70359bf31a
commit 46b7ea7c85
5 changed files with 167 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
examples/quick_start/more_stream_methods.py
View File

@@ -1,44 +0,0 @@
import asyncio
import sglang as sgl
@sgl.function
def multi_turn_question(s, question_1, question_2):
s += sgl.system("You are a helpful assistant.")
s += sgl.user(question_1)
s += sgl.assistant(sgl.gen("answer_1", max_tokens=256))
s += sgl.user(question_2)
s += sgl.assistant(sgl.gen("answer_2", max_tokens=256))
sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo"))
#sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000"))
def stream_a_variable():
state = multi_turn_question.run(
question_1="What is the capital of the United States?",
question_2="List two local attractions.",
stream=True
)
for out in state.text_iter(var_name="answer_2"):
print(out, end="", flush=True)
print()
async def async_stream():
state = multi_turn_question.run(
question_1="What is the capital of the United States?",
question_2="List two local attractions.",
stream=True
)
async for out in state.text_async_iter(var_name="answer_2"):
print(out, end="", flush=True)
print()
if __name__ == "__main__":
#stream_a_variable()
asyncio.run(async_stream())