初始化项目,由ModelHub XC社区提供模型
Model: dphn/dolphin-2.2-mistral-7b Source: Original Platform
This commit is contained in:
41
eval.sh
Normal file
41
eval.sh
Normal file
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
|
||||
MODEL_PATH="cognitivecomputations/dolphin-2.2-mistral-7b"
|
||||
MODEL_NAME="dolphin-2.2-mistral-7b"
|
||||
RESULTS_PATH="/workspace/results/$MODEL_NAME"
|
||||
mkdir -p "$RESULTS_PATH"
|
||||
|
||||
PRETRAINED_ARGS="$MODEL_PATH,tensor_parallel_size=4,dtype=auto,trust_remote_code=True,gpu_memory_utilization=0.8"
|
||||
MODEL_ARGS="pretrained=$PRETRAINED_ARGS"
|
||||
|
||||
tasks=(
|
||||
"mmlu"
|
||||
"truthfulqa"
|
||||
"gsm8k"
|
||||
"hellaswag"
|
||||
"arc_challenge"
|
||||
"winogrande")
|
||||
|
||||
# Function to get the number of fewshot for a given task
|
||||
get_num_fewshot() {
|
||||
case "$1" in
|
||||
"mmlu") echo 5 ;;
|
||||
"truthfulqa") echo 0 ;;
|
||||
"gsm8k") echo 5 ;;
|
||||
"hellaswag") echo 10 ;;
|
||||
"arc_challenge") echo 25 ;;
|
||||
"winogrande") echo 5 ;;
|
||||
*) echo 0 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
for TASK in "${tasks[@]}"; do
|
||||
echo lm_eval --model vllm --model_args "$MODEL_ARGS" --task="$TASK" --num_fewshot "$(get_num_fewshot "$TASK")" --batch_size 8 --output_path "$RESULTS_PATH/$TASK.json"
|
||||
lm_eval --model vllm --model_args "$MODEL_ARGS" --task="$TASK" --num_fewshot "$(get_num_fewshot "$TASK")" --batch_size 8 --output_path "$RESULTS_PATH/$TASK.json"
|
||||
done
|
||||
|
||||
|
||||
jq -s '[.[]]' $RESULTS_PATH/*.json > $RESULTS_PATH/eval_results.json
|
||||
|
||||
huggingface-cli upload cognitivecomputations/$MODEL_NAME $RESULTS_PATH/eval_results.json
|
||||
huggingface-cli upload cognitivecomputations/$MODEL_NAME eval.sh
|
||||
Reference in New Issue
Block a user