Add openAI compatible API (#1810)

Co-authored-by: Chayenne <zhaochenyang@g.ucla.edu>
This commit is contained in:
Chayenne
2024-10-27 10:51:42 -07:00
committed by GitHub
parent eaade87a42
commit 51c81e339b
7 changed files with 800 additions and 56 deletions

View File

@@ -4,7 +4,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"# Quick Start"
"# Quick Start: Launch A Server and Send Requests\n",
"\n",
"This section provides a quick start guide to using SGLang after installation."
]
},
{
@@ -13,12 +15,13 @@
"source": [
"## Launch a server\n",
"\n",
"This code uses `subprocess.Popen` to start an SGLang server process, equivalent to executing \n",
"This code block is equivalent to executing \n",
"\n",
"```bash\n",
"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n",
"--port 30000 --host 0.0.0.0 --log-level warning\n",
"```\n",
"\n",
"in your command line and wait for the server to be ready."
]
},
@@ -39,10 +42,12 @@
"from sglang.utils import execute_shell_command, wait_for_server, terminate_process\n",
"\n",
"\n",
"server_process = execute_shell_command(\"\"\"\n",
"server_process = execute_shell_command(\n",
" \"\"\"\n",
"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n",
"--port 30000 --host 0.0.0.0 --log-level warning\n",
"\"\"\")\n",
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(\"http://localhost:30000\")\n",
"print(\"Server is ready. Proceeding with the next steps.\")"
@@ -105,9 +110,7 @@
"# Always assign an api_key, even if not specified during server initialization.\n",
"# Setting an API key during server initialization is strongly recommended.\n",
"\n",
"client = openai.Client(\n",
" base_url=\"http://127.0.0.1:30000/v1\", api_key=\"None\"\n",
")\n",
"client = openai.Client(base_url=\"http://127.0.0.1:30000/v1\", api_key=\"None\")\n",
"\n",
"# Chat completion example\n",
"\n",