EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix docs (#2164)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-11-24 08:25:56 -08:00
committed by GitHub
parent be0124bda0
commit 8912b7637f
2 changed files with 18 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
docs/backend/offline_engine_api.ipynb
View File

@@ -39,7 +39,6 @@
"# launch the offline engine\n",
"\n",
"import sglang as sgl\n",
"from sglang.utils import print_highlight\n",
"import asyncio\n",
"\n",
"llm = sgl.Engine(model_path=\"meta-llama/Meta-Llama-3.1-8B-Instruct\")"
@@ -69,8 +68,8 @@
"\n",
"outputs = llm.generate(prompts, sampling_params)\n",
"for prompt, output in zip(prompts, outputs):\n",
" print_highlight(\"===============================\")\n",
" print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")"
" print(\"===============================\")\n",
" print(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")"
]
},
{
@@ -93,10 +92,10 @@
"]\n",
"sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n",
"\n",
"print_highlight(\"\\n=== Testing synchronous streaming generation ===\")\n",
"print(\"\\n=== Testing synchronous streaming generation ===\")\n",
"\n",
"for prompt in prompts:\n",
" print_highlight(f\"\\nPrompt: {prompt}\")\n",
" print(f\"\\nPrompt: {prompt}\")\n",
" print(\"Generated text: \", end=\"\", flush=True)\n",
"\n",
" for chunk in llm.generate(prompt, sampling_params, stream=True):\n",
@@ -125,15 +124,15 @@
"\n",
"sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n",
"\n",
"print_highlight(\"\\n=== Testing asynchronous batch generation ===\")\n",
"print(\"\\n=== Testing asynchronous batch generation ===\")\n",
"\n",
"\n",
"async def main():\n",
" outputs = await llm.async_generate(prompts, sampling_params)\n",
"\n",
" for prompt, output in zip(prompts, outputs):\n",
" print_highlight(f\"\\nPrompt: {prompt}\")\n",
" print_highlight(f\"Generated text: {output['text']}\")\n",
" print(f\"\\nPrompt: {prompt}\")\n",
" print(f\"Generated text: {output['text']}\")\n",
"\n",
"\n",
"asyncio.run(main())"
@@ -159,12 +158,12 @@
"]\n",
"sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n",
"\n",
"print_highlight(\"\\n=== Testing asynchronous streaming generation ===\")\n",
"print(\"\\n=== Testing asynchronous streaming generation ===\")\n",
"\n",
"\n",
"async def main():\n",
" for prompt in prompts:\n",
" print_highlight(f\"\\nPrompt: {prompt}\")\n",
" print(f\"\\nPrompt: {prompt}\")\n",
" print(\"Generated text: \", end=\"\", flush=True)\n",
"\n",
" generator = await llm.async_generate(prompt, sampling_params, stream=True)\n",