35 lines
1.1 KiB
Bash
35 lines
1.1 KiB
Bash
# Install required package
|
|
pip install antlr4-python3-runtime==4.11 immutabledict langdetect
|
|
|
|
python -c "import nltk; nltk.download('punkt')"
|
|
|
|
MODEL_PATHS=(
|
|
meta-llama/Meta-Llama-3.1-8B-Instruct
|
|
)
|
|
|
|
for MODEL_PATH in "${MODEL_PATHS[@]}"; do
|
|
MODEL_NAME=$(basename "$MODEL_PATH")
|
|
MODEL_DIR="./results/$MODEL_NAME"
|
|
mkdir -p "$MODEL_DIR"
|
|
|
|
MODEL_ARGS="trust_remote_code=True,pretrained=$MODEL_PATH,dtype=bfloat16"
|
|
|
|
BASE_COMMAND="accelerate launch -m lm_eval --model hf --model_args $MODEL_ARGS --batch_size 4 --fewshot_as_multiturn --apply_chat_template"
|
|
|
|
# IFEval
|
|
$BASE_COMMAND --tasks leaderboard_ifeval --fewshot_as_multiturn --output_path "$MODEL_DIR/ifeval"
|
|
|
|
# BBH (Big-Bench Hard)
|
|
$BASE_COMMAND --tasks leaderboard_bbh --num_fewshot 3 --fewshot_as_multiturn --output_path "$MODEL_DIR/bbh"
|
|
|
|
# GPQA
|
|
$BASE_COMMAND --tasks leaderboard_gpqa --fewshot_as_multiturn --output_path "$MODEL_DIR/gpqa"
|
|
|
|
# MMLU-Pro
|
|
$BASE_COMMAND --tasks leaderboard_mmlu_pro --num_fewshot 5 --fewshot_as_multiturn --output_path "$MODEL_DIR/mmlu_pro"
|
|
|
|
# TruthfulQA
|
|
$BASE_COMMAND --tasks truthfulqa_mc2 --fewshot_as_multiturn --output_path "$MODEL_DIR/truthfulqa"
|
|
|
|
|
|
done |