Support oai in benchmark/mmlu (#323)

2024-03-22 13:37:57 -07:00
parent b57abe1663
commit e2b2f0a213
3 changed files with 15 additions and 5 deletions
--- a/benchmark/mmlu/bench_sglang.py
+++ b/benchmark/mmlu/bench_sglang.py
@@ -64,10 +64,16 @@ def evaluate(args, subject, dev_df, test_df):
    #####################################

    import sglang as sgl
-    
-    @sgl.function
-    def few_shot_mmlu(s, examples, question):
-        s += examples + question + sgl.gen("answer")
+
+    if args.backend.startswith("gpt-"):
+        @sgl.function
+        def few_shot_mmlu(s, examples, question):
+            s += sgl.user(examples + question)
+            s += sgl.assistant(sgl.gen("answer"))
+    else:
+        @sgl.function
+        def few_shot_mmlu(s, examples, question):
+            s += examples + question + sgl.gen("answer")

    #####################################
    ########## SGL Program End ##########