Improve docs & Add JSON decode example (#121)

This commit is contained in:
Lianmin Zheng
2024-01-30 05:45:27 -08:00
committed by GitHub
parent 0617528632
commit 97aa9b3284
19 changed files with 212 additions and 61 deletions

View File

@@ -1,3 +1,7 @@
"""
Usage:
python3 async_io.py
"""
import asyncio
from sglang import Runtime
@@ -27,8 +31,8 @@ async def generate(
if __name__ == "__main__":
runtime = Runtime(model_path="meta-llama/Llama-2-7b-chat-hf")
print("runtime ready")
print("--- runtime ready ---\n")
prompt = "Who is Alan Turing?"
sampling_params = {"max_new_tokens": 128}
asyncio.run(generate(runtime, prompt, sampling_params))