EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Docs: Only use X-Grammar in structed output (#2991)

Browse Source
This commit is contained in:
Chayenne
2025-01-19 20:22:47 -08:00
committed by GitHub
parent cd493b5afc
commit 0ffcfdf474
2 changed files with 22 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
docs/backend/openai_api_completions.ipynb
View File

@@ -41,10 +41,10 @@
")\n", ")\n",
"\n", "\n",
"server_process = execute_shell_command(\n", "server_process = execute_shell_command(\n",
" \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --port 30000 --host 0.0.0.0\"\n", " \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --port 30020 --host 0.0.0.0\"\n",
")\n", ")\n",
"\n", "\n",
"wait_for_server(\"http://localhost:30000\")" "wait_for_server(\"http://localhost:30020\")"
] ]
}, },
{ {
@@ -68,7 +68,7 @@
"source": [ "source": [
"import openai\n", "import openai\n",
"\n", "\n",
"client = openai.Client(base_url=\"http://127.0.0.1:30000/v1\", api_key=\"None\")\n", "client = openai.Client(base_url=\"http://127.0.0.1:30020/v1\", api_key=\"None\")\n",
"\n", "\n",
"response = client.chat.completions.create(\n", "response = client.chat.completions.create(\n",
" model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n",
@@ -214,125 +214,8 @@
"metadata": {}, "metadata": {},
"source": [ "source": [
"## Structured Outputs (JSON, Regex, EBNF)\n", "## Structured Outputs (JSON, Regex, EBNF)\n",
"You can specify a JSON schema, [regular expression](https://en.wikipedia.org/wiki/Regular_expression) or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request.\n",
"\n", "\n",
"SGLang supports two grammar backends:\n", "For OpenAI compatible structed outputs API, refer to [Structured Outputs](https://docs.sglang.ai/backend/structured_outputs.html#OpenAI-Compatible-API) for more details.\n"
"\n",
"- [Outlines](https://github.com/dottxt-ai/outlines) (default): Supports JSON schema and regular expression constraints.\n",
"- [XGrammar](https://github.com/mlc-ai/xgrammar): Supports JSON schema, regular expression, and EBNF constraints.\n",
" - XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md)\n",
"\n",
"Initialize the XGrammar backend using `--grammar-backend xgrammar` flag\n",
"```bash\n",
"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n",
"--port 30000 --host 0.0.0.0 --grammar-backend [xgrammar|outlines] # xgrammar or outlines (default: outlines)\n",
"```\n",
"\n",
"### JSON"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"\n",
"json_schema = json.dumps(\n",
" {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n",
" \"population\": {\"type\": \"integer\"},\n",
" },\n",
" \"required\": [\"name\", \"population\"],\n",
" }\n",
")\n",
"\n",
"response = client.chat.completions.create(\n",
" model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n",
" messages=[\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"Give me the information of the capital of France in the JSON format.\",\n",
" },\n",
" ],\n",
" temperature=0,\n",
" max_tokens=128,\n",
" response_format={\n",
" \"type\": \"json_schema\",\n",
" \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n",
" },\n",
")\n",
"\n",
"print_highlight(response.choices[0].message.content)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Regular expression (use default \"outlines\" backend)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"response = client.chat.completions.create(\n",
" model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"What is the capital of France?\"},\n",
" ],\n",
" temperature=0,\n",
" max_tokens=128,\n",
" extra_body={\"regex\": \"(Paris|London)\"},\n",
")\n",
"\n",
"print_highlight(response.choices[0].message.content)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### EBNF (use \"xgrammar\" backend)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# terminate the existing server(that's using default outlines backend) for this demo\n",
"terminate_process(server_process)\n",
"\n",
"# start new server with xgrammar backend\n",
"server_process = execute_shell_command(\n",
" \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --port 30000 --host 0.0.0.0 --grammar-backend xgrammar\"\n",
")\n",
"wait_for_server(\"http://localhost:30000\")\n",
"\n",
"# EBNF example\n",
"ebnf_grammar = r\"\"\"\n",
" root ::= \"Hello\" | \"Hi\" | \"Hey\"\n",
" \"\"\"\n",
"response = client.chat.completions.create(\n",
" model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n",
" messages=[\n",
" {\"role\": \"system\", \"content\": \"You are a helpful EBNF test bot.\"},\n",
" {\"role\": \"user\", \"content\": \"Say a greeting.\"},\n",
" ],\n",
" temperature=0,\n",
" max_tokens=32,\n",
" extra_body={\"ebnf\": ebnf_grammar},\n",
")\n",
"\n",
"print_highlight(response.choices[0].message.content)"
] ]
}, },
{ {
@@ -362,7 +245,7 @@
"import time\n", "import time\n",
"from openai import OpenAI\n", "from openai import OpenAI\n",
"\n", "\n",
"client = OpenAI(base_url=\"http://127.0.0.1:30000/v1\", api_key=\"None\")\n", "client = OpenAI(base_url=\"http://127.0.0.1:30020/v1\", api_key=\"None\")\n",
"\n", "\n",
"requests = [\n", "requests = [\n",
" {\n", " {\n",
@@ -465,7 +348,7 @@
"import time\n", "import time\n",
"from openai import OpenAI\n", "from openai import OpenAI\n",
"\n", "\n",
"client = OpenAI(base_url=\"http://127.0.0.1:30000/v1\", api_key=\"None\")\n", "client = OpenAI(base_url=\"http://127.0.0.1:30020/v1\", api_key=\"None\")\n",
"\n", "\n",
"requests = []\n", "requests = []\n",
"for i in range(100):\n", "for i in range(100):\n",
@@ -542,7 +425,7 @@
"from openai import OpenAI\n", "from openai import OpenAI\n",
"import os\n", "import os\n",
"\n", "\n",
"client = OpenAI(base_url=\"http://127.0.0.1:30000/v1\", api_key=\"None\")\n", "client = OpenAI(base_url=\"http://127.0.0.1:30020/v1\", api_key=\"None\")\n",
"\n", "\n",
"requests = []\n", "requests = []\n",
"for i in range(500):\n", "for i in range(500):\n",

93
docs/backend/structured_outputs.ipynb
View File

@@ -17,11 +17,12 @@
"\n", "\n",
"- [Outlines](https://github.com/dottxt-ai/outlines) (default): Supports JSON schema and regular expression constraints.\n", "- [Outlines](https://github.com/dottxt-ai/outlines) (default): Supports JSON schema and regular expression constraints.\n",
"- [XGrammar](https://github.com/mlc-ai/xgrammar): Supports JSON schema, regular expression, and EBNF constraints.\n", "- [XGrammar](https://github.com/mlc-ai/xgrammar): Supports JSON schema, regular expression, and EBNF constraints.\n",
" - XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md)\n",
"\n", "\n",
"We suggest using XGrammar whenever possible for its better performance. For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar).\n", "We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar).\n",
"\n", "\n",
"To use Xgrammar, simply add `--grammar-backend` xgrammar when launching the server. If no backend is specified, Outlines will be used as the default." "To use Xgrammar, simply add `--grammar-backend` xgrammar when launching the server. If no backend is specified, Outlines will be used as the default.\n",
"\n",
"For better output quality, **It's advisable to explicitly include instructions in the prompt to guide the model to generate the desired format.** For example, you can specify, 'Please generate the output in the following JSON format: ...'.\n"
] ]
}, },
{ {
@@ -93,7 +94,7 @@
" messages=[\n", " messages=[\n",
" {\n", " {\n",
" \"role\": \"user\",\n", " \"role\": \"user\",\n",
" \"content\": \"Give me the information of the capital of France in the JSON format.\",\n", " \"content\": \"Please generate the information of the capital of France in the JSON format.\",\n",
" },\n", " },\n",
" ],\n", " ],\n",
" temperature=0,\n", " temperature=0,\n",
@@ -197,20 +198,6 @@
"print_highlight(response.choices[0].message.content)" "print_highlight(response.choices[0].message.content)"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"terminate_process(server_process)\n",
"server_process = execute_shell_command(\n",
" \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --port 30000 --host 0.0.0.0\"\n",
")\n",
"\n",
"wait_for_server(\"http://localhost:30000\")"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
@@ -237,15 +224,6 @@
"print_highlight(response.choices[0].message.content)" "print_highlight(response.choices[0].message.content)"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"terminate_process(server_process)"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
@@ -253,21 +231,6 @@
"## Native API and SGLang Runtime (SRT)" "## Native API and SGLang Runtime (SRT)"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"server_process = execute_shell_command(\n",
" \"\"\"\n",
"python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --port=30010 --grammar-backend xgrammar\n",
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(\"http://localhost:30010\")"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
@@ -301,7 +264,7 @@
"\n", "\n",
"# Make API request\n", "# Make API request\n",
"response = requests.post(\n", "response = requests.post(\n",
" \"http://localhost:30010/generate\",\n", " \"http://localhost:30000/generate\",\n",
" json={\n", " json={\n",
" \"text\": \"Here is the information of the capital of France in the JSON format.\\n\",\n", " \"text\": \"Here is the information of the capital of France in the JSON format.\\n\",\n",
" \"sampling_params\": {\n", " \"sampling_params\": {\n",
@@ -346,7 +309,7 @@
"\n", "\n",
"# JSON\n", "# JSON\n",
"response = requests.post(\n", "response = requests.post(\n",
" \"http://localhost:30010/generate\",\n", " \"http://localhost:30000/generate\",\n",
" json={\n", " json={\n",
" \"text\": \"Here is the information of the capital of France in the JSON format.\\n\",\n", " \"text\": \"Here is the information of the capital of France in the JSON format.\\n\",\n",
" \"sampling_params\": {\n", " \"sampling_params\": {\n",
@@ -376,7 +339,7 @@
"import requests\n", "import requests\n",
"\n", "\n",
"response = requests.post(\n", "response = requests.post(\n",
" \"http://localhost:30010/generate\",\n", " \"http://localhost:30000/generate\",\n",
" json={\n", " json={\n",
" \"text\": \"Give me the information of the capital of France.\",\n", " \"text\": \"Give me the information of the capital of France.\",\n",
" \"sampling_params\": {\n", " \"sampling_params\": {\n",
@@ -399,22 +362,6 @@
"print_highlight(response.json())" "print_highlight(response.json())"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"terminate_process(server_process)\n",
"server_process = execute_shell_command(\n",
" \"\"\"\n",
"python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --port=30010\n",
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(\"http://localhost:30010\")"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
@@ -429,7 +376,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"response = requests.post(\n", "response = requests.post(\n",
" \"http://localhost:30010/generate\",\n", " \"http://localhost:30000/generate\",\n",
" json={\n", " json={\n",
" \"text\": \"Paris is the capital of\",\n", " \"text\": \"Paris is the capital of\",\n",
" \"sampling_params\": {\n", " \"sampling_params\": {\n",
@@ -466,7 +413,7 @@
"source": [ "source": [
"import sglang as sgl\n", "import sglang as sgl\n",
"\n", "\n",
"llm_xgrammar = sgl.Engine(\n", "llm = sgl.Engine(\n",
" model_path=\"meta-llama/Meta-Llama-3.1-8B-Instruct\", grammar_backend=\"xgrammar\"\n", " model_path=\"meta-llama/Meta-Llama-3.1-8B-Instruct\", grammar_backend=\"xgrammar\"\n",
")" ")"
] ]
@@ -514,7 +461,7 @@
" \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n", " \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n",
"}\n", "}\n",
"\n", "\n",
"outputs = llm_xgrammar.generate(prompts, sampling_params)\n", "outputs = llm.generate(prompts, sampling_params)\n",
"for prompt, output in zip(prompts, outputs):\n", "for prompt, output in zip(prompts, outputs):\n",
" print_highlight(\"===============================\")\n", " print_highlight(\"===============================\")\n",
" print_highlight(f\"Prompt: {prompt}\") # validate the output by the pydantic model\n", " print_highlight(f\"Prompt: {prompt}\") # validate the output by the pydantic model\n",
@@ -554,7 +501,7 @@
"\n", "\n",
"sampling_params = {\"temperature\": 0.1, \"top_p\": 0.95, \"json_schema\": json_schema}\n", "sampling_params = {\"temperature\": 0.1, \"top_p\": 0.95, \"json_schema\": json_schema}\n",
"\n", "\n",
"outputs = llm_xgrammar.generate(prompts, sampling_params)\n", "outputs = llm.generate(prompts, sampling_params)\n",
"for prompt, output in zip(prompts, outputs):\n", "for prompt, output in zip(prompts, outputs):\n",
" print_highlight(\"===============================\")\n", " print_highlight(\"===============================\")\n",
" print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")"
@@ -591,22 +538,12 @@
" ),\n", " ),\n",
"}\n", "}\n",
"\n", "\n",
"outputs = llm_xgrammar.generate(prompts, sampling_params)\n", "outputs = llm.generate(prompts, sampling_params)\n",
"for prompt, output in zip(prompts, outputs):\n", "for prompt, output in zip(prompts, outputs):\n",
" print_highlight(\"===============================\")\n", " print_highlight(\"===============================\")\n",
" print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"llm_xgrammar.shutdown()\n",
"llm_outlines = sgl.Engine(model_path=\"meta-llama/Meta-Llama-3.1-8B-Instruct\")"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
@@ -627,7 +564,7 @@
"\n", "\n",
"sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95, \"regex\": \"(France|England)\"}\n", "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95, \"regex\": \"(France|England)\"}\n",
"\n", "\n",
"outputs = llm_outlines.generate(prompts, sampling_params)\n", "outputs = llm.generate(prompts, sampling_params)\n",
"for prompt, output in zip(prompts, outputs):\n", "for prompt, output in zip(prompts, outputs):\n",
" print_highlight(\"===============================\")\n", " print_highlight(\"===============================\")\n",
" print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")"
@@ -639,7 +576,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"llm_outlines.shutdown()" "llm.shutdown()"
] ]
} }
], ],