EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix: docs/backend/structured_outputs.ipynb (#4884)

Browse Source
This commit is contained in:
thyecust
2025-04-12 17:18:55 +08:00
committed by GitHub
parent 57de7c6b5f
commit 2074a2e6b6
1 changed files with 35 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
docs/backend/structured_outputs.ipynb
View File

@@ -385,6 +385,10 @@
"import json\n", "import json\n",
"from pydantic import BaseModel, Field\n", "from pydantic import BaseModel, Field\n",
"\n", "\n",
"from transformers import AutoTokenizer\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n",
"\n",
"\n", "\n",
"# Define the schema using Pydantic\n", "# Define the schema using Pydantic\n",
"class CapitalInfo(BaseModel):\n", "class CapitalInfo(BaseModel):\n",
@@ -393,10 +397,19 @@
"\n", "\n",
"\n", "\n",
"# Make API request\n", "# Make API request\n",
"messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"Here is the information of the capital of France in the JSON format.\\n\",\n",
" }\n",
"]\n",
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
")\n",
"response = requests.post(\n", "response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n", " f\"http://localhost:{port}/generate\",\n",
" json={\n", " json={\n",
" \"text\": \"Here is the information of the capital of France in the JSON format.\\n\",\n", " \"text\": text,\n",
" \"sampling_params\": {\n", " \"sampling_params\": {\n",
" \"temperature\": 0,\n", " \"temperature\": 0,\n",
" \"max_new_tokens\": 64,\n", " \"max_new_tokens\": 64,\n",
@@ -441,7 +454,7 @@
"response = requests.post(\n", "response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n", " f\"http://localhost:{port}/generate\",\n",
" json={\n", " json={\n",
" \"text\": \"Here is the information of the capital of France in the JSON format.\\n\",\n", " \"text\": text,\n",
" \"sampling_params\": {\n", " \"sampling_params\": {\n",
" \"temperature\": 0,\n", " \"temperature\": 0,\n",
" \"max_new_tokens\": 64,\n", " \"max_new_tokens\": 64,\n",
@@ -466,10 +479,19 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"Give me the information of the capital of France.\",\n",
" }\n",
"]\n",
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
")\n",
"response = requests.post(\n", "response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n", " f\"http://localhost:{port}/generate\",\n",
" json={\n", " json={\n",
" \"text\": \"Give me the information of the capital of France.\",\n", " \"text\": text,\n",
" \"sampling_params\": {\n", " \"sampling_params\": {\n",
" \"max_new_tokens\": 128,\n", " \"max_new_tokens\": 128,\n",
" \"temperature\": 0,\n", " \"temperature\": 0,\n",
@@ -503,10 +525,19 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"Paris is the capital of\",\n",
" }\n",
"]\n",
"text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n",
")\n",
"response = requests.post(\n", "response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n", " f\"http://localhost:{port}/generate\",\n",
" json={\n", " json={\n",
" \"text\": \"Paris is the capital of\",\n", " \"text\": text,\n",
" \"sampling_params\": {\n", " \"sampling_params\": {\n",
" \"temperature\": 0,\n", " \"temperature\": 0,\n",
" \"max_new_tokens\": 64,\n", " \"max_new_tokens\": 64,\n",