EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[script] update loogle test (#7975)

Browse Source
This commit is contained in:
Ying Sheng
2025-07-12 00:06:17 -07:00
committed by GitHub
parent bcc5ba94b4
commit ccfa084125
1 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
python/sglang/eval/loogle_eval.py
View File

@@ -73,6 +73,8 @@ async def benchmark(args):
tasks: List[asyncio.Task] = [] tasks: List[asyncio.Task] = []
for idx, ex in enumerate(dataset): for idx, ex in enumerate(dataset):
if idx >= args.num_prompts:
break
tasks.append( tasks.append(
asyncio.create_task( asyncio.create_task(
fetch_response( fetch_response(
@@ -103,6 +105,8 @@ def analyse(args):
hyps: List[str] = [] hyps: List[str] = []
refs: List[str] = [] refs: List[str] = []
for idx, ex in enumerate(tqdm(dataset, desc="Loading responses")): for idx, ex in enumerate(tqdm(dataset, desc="Loading responses")):
if idx >= args.num_prompts:
break
pkl_file = output_dir / f"response_{idx}.pkl" pkl_file = output_dir / f"response_{idx}.pkl"
if not pkl_file.exists(): if not pkl_file.exists():
raise FileNotFoundError(pkl_file) raise FileNotFoundError(pkl_file)
@@ -150,6 +154,9 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
"--output-dir", default="tmp-output-dir", help="Directory for cached responses" "--output-dir", default="tmp-output-dir", help="Directory for cached responses"
) )
parser.add_argument(
"--num-prompts", type=int, default=10000, help="Number of prompts to run"
)
args = parser.parse_args() args = parser.parse_args()
asyncio.run(benchmark(args)) asyncio.run(benchmark(args))