EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

feat(pre-commit): trim unnecessary notebook metadata from git history (#2127)

Browse Source
This commit is contained in:
Xuehai Pan
2024-11-23 05:04:51 +08:00
committed by GitHub
parent 62a4a339eb
commit 72f87b723b
8 changed files with 143 additions and 614 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
.pre-commit-config.yaml
View File

@@ -1,6 +1,3 @@
default_language_version:
python: python3.9
default_stages: [pre-commit, pre-push, manual]
repos:
@@ -28,7 +25,11 @@ repos:
- repo: https://github.com/psf/black
rev: 24.10.0
hooks:
- id: black
types: [python]
- id: black-jupyter
types: [jupyter]
- repo: https://github.com/kynan/nbstripout
rev: 0.8.1
hooks:
- id: nbstripout
args:
- '--keep-output'
- '--extra-keys=metadata.kernelspec metadata.language_info.version'

145
docs/backend/native_api.ipynb
View File

@@ -32,14 +32,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:44:42.063503Z",
"iopub.status.busy": "2024-11-07T18:44:42.063379Z",
"iopub.status.idle": "2024-11-07T18:45:07.255300Z",
"shell.execute_reply": "2024-11-07T18:45:07.254547Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"from sglang.utils import (\n",
@@ -71,14 +64,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:07.258292Z",
"iopub.status.busy": "2024-11-07T18:45:07.257710Z",
"iopub.status.idle": "2024-11-07T18:45:07.611559Z",
"shell.execute_reply": "2024-11-07T18:45:07.610842Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"url = \"http://localhost:30010/generate\"\n",
@@ -99,14 +85,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:07.613911Z",
"iopub.status.busy": "2024-11-07T18:45:07.613746Z",
"iopub.status.idle": "2024-11-07T18:45:07.620286Z",
"shell.execute_reply": "2024-11-07T18:45:07.619779Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"url = \"http://localhost:30010/get_server_args\"\n",
@@ -130,14 +109,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:07.622407Z",
"iopub.status.busy": "2024-11-07T18:45:07.622267Z",
"iopub.status.idle": "2024-11-07T18:45:07.628290Z",
"shell.execute_reply": "2024-11-07T18:45:07.627793Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"url = \"http://localhost:30010/get_model_info\"\n",
@@ -162,14 +134,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:07.630585Z",
"iopub.status.busy": "2024-11-07T18:45:07.630235Z",
"iopub.status.idle": "2024-11-07T18:45:07.643498Z",
"shell.execute_reply": "2024-11-07T18:45:07.643007Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"url = \"http://localhost:30010/health_generate\"\n",
@@ -181,14 +146,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:07.645336Z",
"iopub.status.busy": "2024-11-07T18:45:07.645196Z",
"iopub.status.idle": "2024-11-07T18:45:07.650363Z",
"shell.execute_reply": "2024-11-07T18:45:07.649837Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"url = \"http://localhost:30010/health\"\n",
@@ -209,14 +167,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:07.652212Z",
"iopub.status.busy": "2024-11-07T18:45:07.652076Z",
"iopub.status.idle": "2024-11-07T18:45:07.658633Z",
"shell.execute_reply": "2024-11-07T18:45:07.658119Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"# flush cache\n",
@@ -239,14 +190,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:07.660468Z",
"iopub.status.busy": "2024-11-07T18:45:07.660325Z",
"iopub.status.idle": "2024-11-07T18:45:07.666476Z",
"shell.execute_reply": "2024-11-07T18:45:07.665984Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"# get_memory_pool_size\n",
@@ -269,14 +213,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:07.668242Z",
"iopub.status.busy": "2024-11-07T18:45:07.668108Z",
"iopub.status.idle": "2024-11-07T18:45:08.725709Z",
"shell.execute_reply": "2024-11-07T18:45:08.725021Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"# successful update with same architecture and size\n",
@@ -294,14 +231,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:08.727865Z",
"iopub.status.busy": "2024-11-07T18:45:08.727721Z",
"iopub.status.idle": "2024-11-07T18:45:11.165841Z",
"shell.execute_reply": "2024-11-07T18:45:11.165282Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"# failed update with different parameter size\n",
@@ -333,14 +263,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:11.167853Z",
"iopub.status.busy": "2024-11-07T18:45:11.167711Z",
"iopub.status.idle": "2024-11-07T18:45:39.542988Z",
"shell.execute_reply": "2024-11-07T18:45:39.542135Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"terminate_process(server_process)\n",
@@ -358,14 +281,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:39.545416Z",
"iopub.status.busy": "2024-11-07T18:45:39.545005Z",
"iopub.status.idle": "2024-11-07T18:45:39.588793Z",
"shell.execute_reply": "2024-11-07T18:45:39.588054Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"# successful encode for embedding model\n",
@@ -390,14 +306,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:39.590729Z",
"iopub.status.busy": "2024-11-07T18:45:39.590446Z",
"iopub.status.idle": "2024-11-07T18:45:59.660376Z",
"shell.execute_reply": "2024-11-07T18:45:59.659992Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"terminate_process(embedding_process)\n",
@@ -417,14 +326,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:45:59.661779Z",
"iopub.status.busy": "2024-11-07T18:45:59.661641Z",
"iopub.status.idle": "2024-11-07T18:46:00.475726Z",
"shell.execute_reply": "2024-11-07T18:46:00.475269Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"from transformers import AutoTokenizer\n",
@@ -454,15 +356,8 @@
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:46:00.477283Z",
"iopub.status.busy": "2024-11-07T18:46:00.477025Z",
"iopub.status.idle": "2024-11-07T18:46:00.525758Z",
"shell.execute_reply": "2024-11-07T18:46:00.525236Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"terminate_process(reward_process)"
@@ -470,11 +365,6 @@
}
],
"metadata": {
"kernelspec": {
"display_name": "AlphaMeemory",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
@@ -484,8 +374,7 @@
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"pygments_lexer": "ipython3"
}
},
"nbformat": 4,

64
docs/backend/offline_engine_api.ipynb
View File

@@ -33,14 +33,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:46:04.789536Z",
"iopub.status.busy": "2024-11-07T18:46:04.789418Z",
"iopub.status.idle": "2024-11-07T18:46:27.038169Z",
"shell.execute_reply": "2024-11-07T18:46:27.037540Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"# launch the offline engine\n",
@@ -62,14 +55,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:46:27.040005Z",
"iopub.status.busy": "2024-11-07T18:46:27.039872Z",
"iopub.status.idle": "2024-11-07T18:46:30.203840Z",
"shell.execute_reply": "2024-11-07T18:46:30.203368Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"prompts = [\n",
@@ -97,14 +83,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:46:30.205880Z",
"iopub.status.busy": "2024-11-07T18:46:30.205719Z",
"iopub.status.idle": "2024-11-07T18:46:39.256561Z",
"shell.execute_reply": "2024-11-07T18:46:39.255880Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"prompts = [\n",
@@ -135,14 +114,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:46:39.259464Z",
"iopub.status.busy": "2024-11-07T18:46:39.259309Z",
"iopub.status.idle": "2024-11-07T18:46:42.384955Z",
"shell.execute_reply": "2024-11-07T18:46:42.384378Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"prompts = [\n",
@@ -177,14 +149,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:46:42.387431Z",
"iopub.status.busy": "2024-11-07T18:46:42.387279Z",
"iopub.status.idle": "2024-11-07T18:46:51.448572Z",
"shell.execute_reply": "2024-11-07T18:46:51.447781Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"prompts = [\n",
@@ -213,15 +178,8 @@
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:46:51.451177Z",
"iopub.status.busy": "2024-11-07T18:46:51.450952Z",
"iopub.status.idle": "2024-11-07T18:46:51.497530Z",
"shell.execute_reply": "2024-11-07T18:46:51.496850Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"llm.shutdown()"
@@ -229,11 +187,6 @@
}
],
"metadata": {
"kernelspec": {
"display_name": "AlphaMeemory",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
@@ -243,8 +196,7 @@
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"pygments_lexer": "ipython3"
}
},
"nbformat": 4,

127
docs/backend/openai_api_completions.ipynb
View File

@@ -37,14 +37,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:46:54.813876Z",
"iopub.status.busy": "2024-11-07T18:46:54.813741Z",
"iopub.status.idle": "2024-11-07T18:47:24.015527Z",
"shell.execute_reply": "2024-11-07T18:47:24.014987Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"from sglang.utils import (\n",
@@ -77,14 +70,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:24.018153Z",
"iopub.status.busy": "2024-11-07T18:47:24.017755Z",
"iopub.status.idle": "2024-11-07T18:47:25.374821Z",
"shell.execute_reply": "2024-11-07T18:47:25.374397Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import openai\n",
@@ -117,14 +103,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:25.376617Z",
"iopub.status.busy": "2024-11-07T18:47:25.376495Z",
"iopub.status.idle": "2024-11-07T18:47:28.482537Z",
"shell.execute_reply": "2024-11-07T18:47:28.482125Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"response = client.chat.completions.create(\n",
@@ -163,14 +142,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:28.484819Z",
"iopub.status.busy": "2024-11-07T18:47:28.484673Z",
"iopub.status.idle": "2024-11-07T18:47:28.659814Z",
"shell.execute_reply": "2024-11-07T18:47:28.659435Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"stream = client.chat.completions.create(\n",
@@ -196,14 +168,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:28.661844Z",
"iopub.status.busy": "2024-11-07T18:47:28.661710Z",
"iopub.status.idle": "2024-11-07T18:47:30.168922Z",
"shell.execute_reply": "2024-11-07T18:47:30.168600Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"response = client.completions.create(\n",
@@ -232,14 +197,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:30.171319Z",
"iopub.status.busy": "2024-11-07T18:47:30.171176Z",
"iopub.status.idle": "2024-11-07T18:47:33.760113Z",
"shell.execute_reply": "2024-11-07T18:47:33.759713Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"response = client.completions.create(\n",
@@ -271,14 +229,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:33.762729Z",
"iopub.status.busy": "2024-11-07T18:47:33.762590Z",
"iopub.status.idle": "2024-11-07T18:47:34.255316Z",
"shell.execute_reply": "2024-11-07T18:47:34.254907Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import json\n",
@@ -323,14 +274,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:34.257393Z",
"iopub.status.busy": "2024-11-07T18:47:34.257246Z",
"iopub.status.idle": "2024-11-07T18:47:34.413506Z",
"shell.execute_reply": "2024-11-07T18:47:34.413172Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"response = client.chat.completions.create(\n",
@@ -366,14 +310,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:34.414816Z",
"iopub.status.busy": "2024-11-07T18:47:34.414541Z",
"iopub.status.idle": "2024-11-07T18:47:34.431341Z",
"shell.execute_reply": "2024-11-07T18:47:34.431081Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import json\n",
@@ -428,14 +365,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:34.432325Z",
"iopub.status.busy": "2024-11-07T18:47:34.432208Z",
"iopub.status.idle": "2024-11-07T18:47:37.444337Z",
"shell.execute_reply": "2024-11-07T18:47:37.444000Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"while batch_response.status not in [\"completed\", \"failed\", \"cancelled\"]:\n",
@@ -483,14 +413,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:47:37.445894Z",
"iopub.status.busy": "2024-11-07T18:47:37.445744Z",
"iopub.status.idle": "2024-11-07T18:48:02.482532Z",
"shell.execute_reply": "2024-11-07T18:48:02.482042Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import json\n",
@@ -566,14 +489,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:48:02.485206Z",
"iopub.status.busy": "2024-11-07T18:48:02.485064Z",
"iopub.status.idle": "2024-11-07T18:48:15.521489Z",
"shell.execute_reply": "2024-11-07T18:48:15.521156Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import json\n",
@@ -660,15 +576,8 @@
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:48:15.522794Z",
"iopub.status.busy": "2024-11-07T18:48:15.522657Z",
"iopub.status.idle": "2024-11-07T18:48:16.875740Z",
"shell.execute_reply": "2024-11-07T18:48:16.874847Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"terminate_process(server_process)"
@@ -676,11 +585,6 @@
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
@@ -690,8 +594,7 @@
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"pygments_lexer": "ipython3"
}
},
"nbformat": 4,

64
docs/backend/openai_api_embeddings.ipynb
View File

@@ -33,14 +33,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:48:21.128020Z",
"iopub.status.busy": "2024-11-07T18:48:21.127898Z",
"iopub.status.idle": "2024-11-07T18:48:45.310371Z",
"shell.execute_reply": "2024-11-07T18:48:45.309469Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"from sglang.utils import (\n",
@@ -70,14 +63,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:48:45.313506Z",
"iopub.status.busy": "2024-11-07T18:48:45.313123Z",
"iopub.status.idle": "2024-11-07T18:48:45.364918Z",
"shell.execute_reply": "2024-11-07T18:48:45.364155Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import subprocess, json\n",
@@ -104,14 +90,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:48:45.367776Z",
"iopub.status.busy": "2024-11-07T18:48:45.367490Z",
"iopub.status.idle": "2024-11-07T18:48:45.411386Z",
"shell.execute_reply": "2024-11-07T18:48:45.411134Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
@@ -138,14 +117,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:48:45.412462Z",
"iopub.status.busy": "2024-11-07T18:48:45.412351Z",
"iopub.status.idle": "2024-11-07T18:48:45.768796Z",
"shell.execute_reply": "2024-11-07T18:48:45.768406Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import openai\n",
@@ -174,14 +146,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:48:45.770227Z",
"iopub.status.busy": "2024-11-07T18:48:45.770106Z",
"iopub.status.idle": "2024-11-07T18:48:47.447065Z",
"shell.execute_reply": "2024-11-07T18:48:47.446733Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import json\n",
@@ -205,15 +170,8 @@
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:48:47.448510Z",
"iopub.status.busy": "2024-11-07T18:48:47.448337Z",
"iopub.status.idle": "2024-11-07T18:48:47.743336Z",
"shell.execute_reply": "2024-11-07T18:48:47.742276Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"terminate_process(embedding_process)"
@@ -221,11 +179,6 @@
}
],
"metadata": {
"kernelspec": {
"display_name": "AlphaMeemory",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
@@ -235,8 +188,7 @@
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"pygments_lexer": "ipython3"
}
},
"nbformat": 4,

64
docs/backend/openai_api_vision.ipynb
View File

@@ -37,14 +37,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:43:47.311708Z",
"iopub.status.busy": "2024-11-07T18:43:47.311517Z",
"iopub.status.idle": "2024-11-07T18:44:18.512576Z",
"shell.execute_reply": "2024-11-07T18:44:18.511909Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"from sglang.utils import (\n",
@@ -76,14 +69,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:44:18.515678Z",
"iopub.status.busy": "2024-11-07T18:44:18.515314Z",
"iopub.status.idle": "2024-11-07T18:44:22.880793Z",
"shell.execute_reply": "2024-11-07T18:44:22.880303Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import subprocess\n",
@@ -127,14 +113,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:44:22.883309Z",
"iopub.status.busy": "2024-11-07T18:44:22.883160Z",
"iopub.status.idle": "2024-11-07T18:44:27.048810Z",
"shell.execute_reply": "2024-11-07T18:44:27.048074Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
@@ -174,14 +153,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:44:27.051312Z",
"iopub.status.busy": "2024-11-07T18:44:27.051190Z",
"iopub.status.idle": "2024-11-07T18:44:32.358097Z",
"shell.execute_reply": "2024-11-07T18:44:32.357628Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"from openai import OpenAI\n",
@@ -225,14 +197,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:44:32.359532Z",
"iopub.status.busy": "2024-11-07T18:44:32.359413Z",
"iopub.status.idle": "2024-11-07T18:44:36.164664Z",
"shell.execute_reply": "2024-11-07T18:44:36.164005Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"from openai import OpenAI\n",
@@ -273,15 +238,8 @@
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:44:36.167123Z",
"iopub.status.busy": "2024-11-07T18:44:36.166535Z",
"iopub.status.idle": "2024-11-07T18:44:37.743761Z",
"shell.execute_reply": "2024-11-07T18:44:37.742510Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"terminate_process(embedding_process)"
@@ -307,11 +265,6 @@
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
@@ -321,8 +274,7 @@
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"pygments_lexer": "ipython3"
}
},
"nbformat": 4,

82
docs/start/send_request.ipynb
View File

@@ -31,14 +31,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:48:52.032229Z",
"iopub.status.busy": "2024-11-07T18:48:52.032105Z",
"iopub.status.idle": "2024-11-07T18:49:20.226042Z",
"shell.execute_reply": "2024-11-07T18:49:20.225562Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"from sglang.utils import (\n",
@@ -68,14 +61,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:49:20.228006Z",
"iopub.status.busy": "2024-11-07T18:49:20.227572Z",
"iopub.status.idle": "2024-11-07T18:49:20.469885Z",
"shell.execute_reply": "2024-11-07T18:49:20.469518Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import subprocess, json\n",
@@ -99,14 +85,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:49:20.471956Z",
"iopub.status.busy": "2024-11-07T18:49:20.471811Z",
"iopub.status.idle": "2024-11-07T18:49:20.667997Z",
"shell.execute_reply": "2024-11-07T18:49:20.667630Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
@@ -132,14 +111,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:49:20.669977Z",
"iopub.status.busy": "2024-11-07T18:49:20.669826Z",
"iopub.status.idle": "2024-11-07T18:49:22.004855Z",
"shell.execute_reply": "2024-11-07T18:49:22.004472Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import openai\n",
@@ -167,14 +139,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:49:22.006983Z",
"iopub.status.busy": "2024-11-07T18:49:22.006858Z",
"iopub.status.idle": "2024-11-07T18:49:23.029098Z",
"shell.execute_reply": "2024-11-07T18:49:23.028697Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import openai\n",
@@ -210,14 +175,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:49:23.031712Z",
"iopub.status.busy": "2024-11-07T18:49:23.031571Z",
"iopub.status.idle": "2024-11-07T18:49:23.787752Z",
"shell.execute_reply": "2024-11-07T18:49:23.787368Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
@@ -246,14 +204,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:49:23.789840Z",
"iopub.status.busy": "2024-11-07T18:49:23.789702Z",
"iopub.status.idle": "2024-11-07T18:49:24.545631Z",
"shell.execute_reply": "2024-11-07T18:49:24.545241Z"
}
},
"metadata": {},
"outputs": [],
"source": [
"import requests, json\n",
@@ -285,15 +236,8 @@
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"execution": {
"iopub.execute_input": "2024-11-07T18:49:24.547641Z",
"iopub.status.busy": "2024-11-07T18:49:24.547497Z",
"iopub.status.idle": "2024-11-07T18:49:25.888864Z",
"shell.execute_reply": "2024-11-07T18:49:25.888114Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"terminate_process(server_process)"
@@ -301,11 +245,6 @@
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
@@ -315,8 +254,7 @@
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"pygments_lexer": "ipython3"
}
},
"nbformat": 4,

198
examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb
View File

@@ -2,6 +2,7 @@
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# RAG Powered by SGLang & Chroma Evaluated using Parea\n",
"\n",
@@ -14,59 +15,49 @@
"ℹ️ This notebook requires an OpenAI API key.\n",
"\n",
"ℹ️ This notebook requires a Parea API key, which can be created [here](https://docs.parea.ai/api-reference/authentication#parea-api-key)."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setting up the environment\n",
"\n",
"We will first install the necessary packages: `sglang`, `parea-ai` and `chromadb`."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# note, if you use a Mac M1 chip, you might need to install grpcio 1.59.0 first such that installing chromadb works\n",
"# !pip install grpcio==1.59.0\n",
"\n",
"!pip install sglang[openai] parea-ai chromadb"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Create a Parea API key as outlined [here](https://docs.parea.ai/api-reference/authentication#parea-api-key) and save it in a `.env` file as `PAREA_API_KEY=your-api-key`."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Indexing the data\n",
"\n",
"Now it's time to download the data & index it! For that, we create a collection called `contexts` in Chroma and add the contexts as documents."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
@@ -90,25 +81,21 @@
" documents=[qca[\"context\"] for qca in question_context_answers],\n",
" ids=[str(i) for i in range(len(question_context_answers))],\n",
" )"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Defining the RAG pipeline\n",
"\n",
"We will start with importing the necessary packages, setting up tracing of OpenAI calls via Parea and setting OpenAI as the default backend for SGLang."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
@@ -129,45 +116,37 @@
"p.integrate_with_sglang()\n",
"\n",
"set_default_backend(OpenAI(\"gpt-3.5-turbo\"))"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now we can define our retrieval step shown below. Notice, the `trace` decorator which will automatically trace inputs, output, latency, etc. of that call."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"@trace\n",
"def retrieval(question: str) -> List[str]:\n",
" return collection.query(query_texts=[question], n_results=1)[\"documents\"][0]"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Next we will define the generation step which uses SGLang to execute the LLM call."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"@function\n",
@@ -185,32 +164,27 @@
" while not state.stream_executor.is_finished:\n",
" time.sleep(1)\n",
" return state.stream_executor.variables[\"answer\"]"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Finally, we can tie it together and execute a sample query."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": "'The World Health Organization formally declared an end to the COVID-19 global health emergency'"
"text/plain": [
"'The World Health Organization formally declared an end to the COVID-19 global health emergency'"
]
},
"execution_count": 3,
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
@@ -229,6 +203,7 @@
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Debug Trace\n",
"\n",
@@ -240,13 +215,11 @@
"When opening the generation step in the playground and rerunning the prompt with max. tokens set to 1000, the correct answer is produced.\n",
"\n",
"![RAG Trace](https://drive.google.com/uc?id=1QI243ogGjzbO01tUrR72g9rFoGzUJqVH)"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Evaluating RAG Pipelines\n",
"\n",
@@ -262,14 +235,12 @@
"To use these evaluation metrics, we can import them from `parea.evals.rag` and `parea.evals.general` and apply them to a function by specifying in the `trace` decorator which evaluation metrics to use. The `@trace` decorator will automatically log the results of the evaluation metrics to the Parea dashboard.\n",
"\n",
"Applying them to the retrieval step:"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from parea.evals.rag import (\n",
@@ -285,23 +256,19 @@
"@trace(eval_funcs=[context_relevancy_eval, percent_target_supported_by_context])\n",
"def retrieval(question: str) -> List[str]:\n",
" return collection.query(query_texts=[question], n_results=1)[\"documents\"][0]"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now we can apply `answer_context_faithfulness` and `answer_matches_target` to the generation step."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from parea.evals.general import answer_matches_target_llm_grader_factory\n",
@@ -327,29 +294,27 @@
" while not state.stream_executor.is_finished:\n",
" time.sleep(1)\n",
" return state.stream_executor.variables[\"answer\"]"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Finally, we tie them together & execute the original sample query."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": "'The World Health Organization formally declared an end to the COVID-19 global health emergency in May 2023.'"
"text/plain": [
"'The World Health Organization formally declared an end to the COVID-19 global health emergency in May 2023.'"
]
},
"execution_count": 4,
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
@@ -364,38 +329,32 @@
"rag_pipeline(\n",
" \"When did the World Health Organization formally declare an end to the COVID-19 global health emergency?\"\n",
")"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Great, the answer is correct! Can you spot the line where we fixed the output truncation issue?\n",
"\n",
"The evaluation scores appear in the bottom right of the logs (screenshot below). Note, that there is no score for `answer_matches_target_llm_grader` and `percent_target_supported_by_context` as these evals are automatically skipped if the target answer is not provided.\n",
"\n",
"![Fixed Max. Tokens](max-tokens-fixed-rag-trace.png)"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Running an experiment\n",
"\n",
"Now we are (almost) ready to evaluate the performance of our RAG pipeline on the entire dataset. First, we will need to apply the `nest_asyncio` package to avoid issues with the Jupyter notebook event loop."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
@@ -410,23 +369,19 @@
"import nest_asyncio\n",
"\n",
"nest_asyncio.apply()"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Running the actual experiment is straight-forward. For that we use `p.experiment` to initialize the experiment with a name, the data (list of key-value pairs fed into our entry function) and the entry function. We then call `run` on the experiment to execute it. Note, that `target` is a reserved key in the data dictionary and will be used as the target answer for evaluation."
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
@@ -478,13 +433,11 @@
" ],\n",
" func=rag_pipeline,\n",
").run()"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Analyzing the results\n",
"\n",
@@ -495,27 +448,17 @@
"Note, above link isn't publicly accessible but the experiment can be accessed through [here](https://app.parea.ai/public-experiments/parea/rag_sglang/30f0244a-d56c-44ff-bdfb-8f47626304b6).\n",
"\n",
"![Experiment Results](https://drive.google.com/uc?id=1KMtJBU47nPB02Pvv3SPPTK7RnHRh5YdA)"
],
"metadata": {
"collapsed": false
}
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [],
"metadata": {
"collapsed": false
}
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
@@ -525,8 +468,7 @@
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
"pygments_lexer": "ipython2"
}
},
"nbformat": 4,