[benchmark] Add benchmark scripts for ceval and boolq (#8946)
Co-authored-by: chenyuxing <2818499974@qq.com> Co-authored-by: hanqing <huang010706@126.com> Co-authored-by: Muggle <62579327+trawolf@users.noreply.github.com> Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
26
benchmark/boolq/parquet_to_json.sh
Executable file
26
benchmark/boolq/parquet_to_json.sh
Executable file
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
|
||||
#define input and output direction
|
||||
input_dir="./boolq/data"
|
||||
output_dir="./boolq/data"
|
||||
|
||||
#define files needed to be handled
|
||||
files=(
|
||||
"train-00000-of-00001.parquet"
|
||||
"validation-00000-of-00001.parquet"
|
||||
)
|
||||
|
||||
#foe files above, use python script to convert the form
|
||||
for file in "${files[@]}"; do
|
||||
input_file="${input_dir}/${file}"
|
||||
output_file="${output_dir}/${file%.parquet}.json"
|
||||
|
||||
echo "Converting ${input_file} to ${output_file} ..."
|
||||
python3 convert_parquet_to_json.py "${input_file}" "${output_file}"
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "Conversion successful: ${output_file}"
|
||||
else
|
||||
echo "Conversion failed: ${input_file}"
|
||||
fi
|
||||
done
|
||||
Reference in New Issue
Block a user