minor: add human eval (#1754)

2024-11-01 14:29:20 +08:00
parent 16eb33ffe2
commit d86a2d6562
2 changed files with 129 additions and 3 deletions
--- a/.github/workflows/nightly-eval.yml
+++ b/.github/workflows/nightly-eval.yml
@@ -25,9 +25,10 @@ jobs:
      - name: Install dependencies
        run: |
          bash scripts/ci_install_dependency.sh
+          pip install --upgrade "evalplus[vllm] @ git+https://github.com/evalplus/evalplus"

-      - name: Nightly gsm8k Accuracy
-        timeout-minutes: 60
+      - name: Nightly gsm8k and human eval Accuracy
+        timeout-minutes: 120
        run: |
          cd test/srt
-          python3 test_nightly_gsm8k_eval.py
+          python3 test_nightly_human_eval.py