RefalMachine/RuadaptQwen2.5-1.5B-instruct
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: RefalMachine/RuadaptQwen2.5-1.5B-instruct
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-09-27 01:39:20 +08:00
commit 4b07714e73
76 changed files with 591981 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8476
llmtf_eval/NEREL.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

54
llmtf_eval/NEREL_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 128,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 200,
"method": "generate"
}
}

10
llmtf_eval/NEREL_total.jsonl Normal file
View File

@@ -0,0 +1,10 @@
{
"task_name": "NEREL",
"results": {
"tp": 2.0,
"fp": 27.0,
"fn": 519.0,
"micro-f1": 0.00727272727272595
},
"leaderboard_result": 0.00727272727272595
}

9321
llmtf_eval/daru_treewayabstractive.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

54
llmtf_eval/daru_treewayabstractive_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 512,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 200,
"method": "generate"
}
}

8
llmtf_eval/daru_treewayabstractive_total.jsonl Normal file
View File

@@ -0,0 +1,8 @@
{
"task_name": "daru/treewayabstractive",
"results": {
"rouge1": 0.3138417117532064,
"rouge2": 0.10462617373556911
},
"leaderboard_result": 0.20923394274438775
}

3
llmtf_eval/daru_treewayextractive.jsonl Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:75cf70167322406bd8c1997ecfc9391b800f34de8cf75514b3d5394666e94263
size 212316130

54
llmtf_eval/daru_treewayextractive_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 1,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"use_flash_attention_2": true,
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 2,
"max_sample_per_dataset": 1000,
"method": "calculate_logsoftmax"
}
}

7
llmtf_eval/daru_treewayextractive_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "daru/treewayextractive",
"results": {
"r-prec": 0.3782488455988456
},
"leaderboard_result": 0.3782488455988456
}

57066
llmtf_eval/darumeru_MultiQ.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

54
llmtf_eval/darumeru_MultiQ_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 64,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 10000000000000,
"method": "generate"
}
}

8
llmtf_eval/darumeru_MultiQ_total.jsonl Normal file
View File

@@ -0,0 +1,8 @@
{
"task_name": "darumeru/MultiQ",
"results": {
"f1": 0.3016876852043635,
"em": 0.21319311663479923
},
"leaderboard_result": 0.25744040091958137
}

8200
llmtf_eval/darumeru_PARus.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/darumeru_PARus_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 64,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 10000000000000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/darumeru_PARus_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "darumeru/PARus",
"results": {
"acc": 0.44
},
"leaderboard_result": 0.44
}

9680
llmtf_eval/darumeru_RCB.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/darumeru_RCB_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 64,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 10000000000000,
"method": "calculate_tokens_proba"
}
}

8
llmtf_eval/darumeru_RCB_total.jsonl Normal file
View File

@@ -0,0 +1,8 @@
{
"task_name": "darumeru/RCB",
"results": {
"acc": 0.4590909090909091,
"f1_macro": 0.36910715356478985
},
"leaderboard_result": 0.4140990313278495
}

7956
llmtf_eval/darumeru_RWSD.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/darumeru_RWSD_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 64,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 10000000000000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/darumeru_RWSD_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "darumeru/RWSD",
"results": {
"acc": 0.49019607843137253
},
"leaderboard_result": 0.49019607843137253
}

38700
llmtf_eval/darumeru_USE.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

54
llmtf_eval/darumeru_USE_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 64,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 10000000000000,
"method": "generate"
}
}

7
llmtf_eval/darumeru_USE_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "darumeru/USE",
"results": {
"grade_norm": 0.06078431372549018
},
"leaderboard_result": 0.06078431372549018
}

19337
llmtf_eval/darumeru_cp_para_ru.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/darumeru_cp_para_ru_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 1024,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 10000000000000,
"method": "generate"
}
}

10
llmtf_eval/darumeru_cp_para_ru_total.jsonl Normal file
View File

@@ -0,0 +1,10 @@
{
"task_name": "darumeru/cp_para_ru",
"results": {
"tokens_per_word": 1.905314928817744,
"symbol_per_token": 3.913951487866651,
"len": 0.9904780330832407,
"lcs": 0.8
},
"leaderboard_result": 0.8
}

104760
llmtf_eval/darumeru_ruOpenBookQA.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/darumeru_ruOpenBookQA_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 64,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"use_flash_attention_2": true,
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 2,
"max_sample_per_dataset": 10000000000000,
"method": "calculate_tokens_proba"
}
}

8
llmtf_eval/darumeru_ruOpenBookQA_total.jsonl Normal file
View File

@@ -0,0 +1,8 @@
{
"task_name": "darumeru/ruOpenBookQA",
"results": {
"acc": 0.615979381443299,
"f1_macro": 0.6154023944317246
},
"leaderboard_result": 0.6156908879375118
}

5040
llmtf_eval/darumeru_ruWorldTree.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/darumeru_ruWorldTree_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 64,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"use_flash_attention_2": true,
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 2,
"max_sample_per_dataset": 10000000000000,
"method": "calculate_tokens_proba"
}
}

8
llmtf_eval/darumeru_ruWorldTree_total.jsonl Normal file
View File

@@ -0,0 +1,8 @@
{
"task_name": "darumeru/ruWorldTree",
"results": {
"acc": 0.7714285714285715,
"f1_macro": 0.7726851851851853
},
"leaderboard_result": 0.7720568783068784
}

272
llmtf_eval/evaluation_log.txt Normal file
View File

@@ -0,0 +1,272 @@
INFO: 2024-11-26 20:17:25,790: llmtf.base.evaluator: Starting eval on ['darumeru/multiq', 'darumeru/parus', 'darumeru/rcb', 'darumeru/rwsd', 'darumeru/use']
INFO: 2024-11-26 20:17:25,791: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:17:25,791: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:17:27,525: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/rummlu']
INFO: 2024-11-26 20:17:27,525: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:17:27,525: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:17:29,517: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/enmmlu']
INFO: 2024-11-26 20:17:29,517: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:17:29,517: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:17:30,060: llmtf.base.darumeru/MultiQ: Loading Dataset: 4.27s
INFO: 2024-11-26 20:17:31,597: llmtf.base.evaluator: Starting eval on ['daru/treewayabstractive']
INFO: 2024-11-26 20:17:31,597: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:17:31,597: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:17:33,345: llmtf.base.evaluator: Starting eval on ['darumeru/cp_para_ru']
INFO: 2024-11-26 20:17:33,345: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:17:33,345: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:17:35,521: llmtf.base.evaluator: Starting eval on ['vikhrmodels/habr_qa_sbs', 'ruparam', 'shlepa/moviesmc', 'shlepa/musicmc', 'shlepa/lawmc', 'shlepa/booksmc']
INFO: 2024-11-26 20:17:35,521: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:17:35,521: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:17:36,308: llmtf.base.darumeru/cp_para_ru: Loading Dataset: 2.96s
INFO: 2024-11-26 20:17:36,742: llmtf.base.daru/treewayabstractive: Loading Dataset: 5.14s
INFO: 2024-11-26 20:17:37,659: llmtf.base.evaluator: Starting eval on ['ruopinionne']
INFO: 2024-11-26 20:17:37,659: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:17:37,660: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:17:37,984: llmtf.base.ruopinionne: Loading Dataset: 0.32s
INFO: 2024-11-26 20:17:39,551: llmtf.base.evaluator: Starting eval on ['nerel']
INFO: 2024-11-26 20:17:39,551: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:17:39,551: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:17:43,423: llmtf.base.NEREL: Loading Dataset: 3.87s
INFO: 2024-11-26 20:17:48,557: llmtf.base.vikhrmodels/habr_qa_sbs: Loading Dataset: 13.04s
INFO: 2024-11-26 20:18:44,135: llmtf.base.darumeru/cp_para_ru: Processing Dataset: 67.83s
INFO: 2024-11-26 20:18:44,145: llmtf.base.darumeru/cp_para_ru: Results for darumeru/cp_para_ru:
INFO: 2024-11-26 20:18:44,149: llmtf.base.darumeru/cp_para_ru: {'tokens_per_word': 1.905314928817744, 'symbol_per_token': 3.913951487866651, 'len': 0.9904780330832407, 'lcs': 0.8}
INFO: 2024-11-26 20:18:44,150: llmtf.base.evaluator: Ended eval
INFO: 2024-11-26 20:18:44,152: llmtf.base.evaluator:
mean darumeru/cp_para_ru
0.800 0.800
INFO: 2024-11-26 20:18:50,857: llmtf.base.NEREL: Processing Dataset: 67.43s
INFO: 2024-11-26 20:18:50,860: llmtf.base.NEREL: Results for NEREL:
INFO: 2024-11-26 20:18:50,864: llmtf.base.NEREL: {'tp': 2.0, 'fp': 27.0, 'fn': 519.0, 'micro-f1': 0.00727272727272595}
INFO: 2024-11-26 20:18:50,865: llmtf.base.evaluator: Ended eval
INFO: 2024-11-26 20:18:50,869: llmtf.base.evaluator:
mean NEREL darumeru/cp_para_ru
0.404 0.007 0.800
INFO: 2024-11-26 20:18:57,627: llmtf.base.daru/treewayabstractive: Processing Dataset: 80.88s
INFO: 2024-11-26 20:18:57,629: llmtf.base.daru/treewayabstractive: Results for daru/treewayabstractive:
INFO: 2024-11-26 20:18:57,632: llmtf.base.daru/treewayabstractive: {'rouge1': 0.3138417117532064, 'rouge2': 0.10462617373556911}
INFO: 2024-11-26 20:18:57,634: llmtf.base.evaluator: Ended eval
INFO: 2024-11-26 20:18:57,637: llmtf.base.evaluator:
mean NEREL daru/treewayabstractive darumeru/cp_para_ru
0.339 0.007 0.209 0.800
INFO: 2024-11-26 20:19:13,379: llmtf.base.darumeru/MultiQ: Processing Dataset: 103.32s
INFO: 2024-11-26 20:19:13,384: llmtf.base.darumeru/MultiQ: Results for darumeru/MultiQ:
INFO: 2024-11-26 20:19:13,404: llmtf.base.darumeru/MultiQ: {'f1': 0.3016876852043635, 'em': 0.21319311663479923}
INFO: 2024-11-26 20:19:13,412: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:19:13,412: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:19:15,573: llmtf.base.darumeru/PARus: Loading Dataset: 2.16s
INFO: 2024-11-26 20:19:19,861: llmtf.base.darumeru/PARus: Processing Dataset: 4.29s
INFO: 2024-11-26 20:19:19,867: llmtf.base.darumeru/PARus: Results for darumeru/PARus:
INFO: 2024-11-26 20:19:19,887: llmtf.base.darumeru/PARus: {'acc': 0.44}
INFO: 2024-11-26 20:19:19,888: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:19:19,888: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:19:22,295: llmtf.base.darumeru/RCB: Loading Dataset: 2.40s
INFO: 2024-11-26 20:19:23,604: llmtf.base.ruopinionne: Processing Dataset: 105.62s
INFO: 2024-11-26 20:19:23,610: llmtf.base.ruopinionne: Results for ruopinionne:
INFO: 2024-11-26 20:19:23,639: llmtf.base.ruopinionne: {'f1': 0.02701209922104298}
INFO: 2024-11-26 20:19:23,640: llmtf.base.evaluator: Ended eval
INFO: 2024-11-26 20:19:23,656: llmtf.base.evaluator:
mean NEREL daru/treewayabstractive darumeru/MultiQ darumeru/PARus darumeru/cp_para_ru ruopinionne
0.290 0.007 0.209 0.257 0.440 0.800 0.027
INFO: 2024-11-26 20:19:27,714: llmtf.base.darumeru/RCB: Processing Dataset: 5.42s
INFO: 2024-11-26 20:19:27,715: llmtf.base.darumeru/RCB: Results for darumeru/RCB:
INFO: 2024-11-26 20:19:27,722: llmtf.base.darumeru/RCB: {'acc': 0.4590909090909091, 'f1_macro': 0.36910715356478985}
INFO: 2024-11-26 20:19:27,724: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:19:27,724: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:19:29,690: llmtf.base.darumeru/RWSD: Loading Dataset: 1.96s
INFO: 2024-11-26 20:19:34,850: llmtf.base.darumeru/RWSD: Processing Dataset: 5.16s
INFO: 2024-11-26 20:19:34,855: llmtf.base.darumeru/RWSD: Results for darumeru/RWSD:
INFO: 2024-11-26 20:19:34,858: llmtf.base.darumeru/RWSD: {'acc': 0.49019607843137253}
INFO: 2024-11-26 20:19:34,859: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:19:34,859: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:19:38,002: llmtf.base.darumeru/USE: Loading Dataset: 3.14s
INFO: 2024-11-26 20:19:52,797: llmtf.base.nlpcoreteam/enMMLU: Loading Dataset: 143.28s
INFO: 2024-11-26 20:19:55,208: llmtf.base.nlpcoreteam/ruMMLU: Loading Dataset: 147.68s
INFO: 2024-11-26 20:20:33,495: llmtf.base.vikhrmodels/habr_qa_sbs: Processing Dataset: 164.94s
INFO: 2024-11-26 20:20:33,496: llmtf.base.vikhrmodels/habr_qa_sbs: Results for vikhrmodels/habr_qa_sbs:
INFO: 2024-11-26 20:20:33,533: llmtf.base.vikhrmodels/habr_qa_sbs: {'acc': 0.547, 'f1_macro': 0.5280856541414141}
INFO: 2024-11-26 20:20:33,546: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:20:33,547: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:20:40,434: llmtf.base.ruparam: Loading Dataset: 6.89s
INFO: 2024-11-26 20:21:07,899: llmtf.base.darumeru/USE: Processing Dataset: 89.90s
INFO: 2024-11-26 20:21:07,900: llmtf.base.darumeru/USE: Results for darumeru/USE:
INFO: 2024-11-26 20:21:07,942: llmtf.base.darumeru/USE: {'grade_norm': 0.06078431372549018}
INFO: 2024-11-26 20:21:07,948: llmtf.base.evaluator: Ended eval
INFO: 2024-11-26 20:21:08,016: llmtf.base.evaluator:
mean NEREL daru/treewayabstractive darumeru/MultiQ darumeru/PARus darumeru/RCB darumeru/RWSD darumeru/USE darumeru/cp_para_ru ruopinionne vikhrmodels/habr_qa_sbs
0.324 0.007 0.209 0.257 0.440 0.414 0.490 0.061 0.800 0.027 0.538
INFO: 2024-11-26 20:25:43,772: llmtf.base.ruparam: Processing Dataset: 303.34s
INFO: 2024-11-26 20:25:43,788: llmtf.base.ruparam: Results for ruparam:
INFO: 2024-11-26 20:25:44,038: llmtf.base.ruparam: {'acc': 0.21363220494053065}
INFO: 2024-11-26 20:25:44,053: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:25:44,053: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:25:47,874: llmtf.base.shlepa/movie_mc: Loading Dataset: 3.82s
INFO: 2024-11-26 20:26:05,103: llmtf.base.shlepa/movie_mc: Processing Dataset: 17.22s
INFO: 2024-11-26 20:26:05,119: llmtf.base.shlepa/movie_mc: Results for shlepa/movie_mc:
INFO: 2024-11-26 20:26:05,122: llmtf.base.shlepa/movie_mc: {'acc': 0.22453703703703703}
INFO: 2024-11-26 20:26:05,130: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:26:05,130: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:26:08,736: llmtf.base.shlepa/music_mc: Loading Dataset: 3.60s
INFO: 2024-11-26 20:26:26,413: llmtf.base.shlepa/music_mc: Processing Dataset: 17.68s
INFO: 2024-11-26 20:26:26,416: llmtf.base.shlepa/music_mc: Results for shlepa/music_mc:
INFO: 2024-11-26 20:26:26,435: llmtf.base.shlepa/music_mc: {'acc': 0.24468085106382978}
INFO: 2024-11-26 20:26:26,438: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:26:26,439: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:26:31,058: llmtf.base.shlepa/law_mc: Loading Dataset: 4.62s
INFO: 2024-11-26 20:27:15,973: llmtf.base.shlepa/law_mc: Processing Dataset: 44.91s
INFO: 2024-11-26 20:27:15,980: llmtf.base.shlepa/law_mc: Results for shlepa/law_mc:
INFO: 2024-11-26 20:27:16,000: llmtf.base.shlepa/law_mc: {'acc': 0.537590113285273}
INFO: 2024-11-26 20:27:16,006: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
INFO: 2024-11-26 20:27:16,006: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
INFO: 2024-11-26 20:27:19,668: llmtf.base.shlepa/books_mc: Loading Dataset: 3.66s
INFO: 2024-11-26 20:27:39,200: llmtf.base.shlepa/books_mc: Processing Dataset: 19.53s
INFO: 2024-11-26 20:27:39,204: llmtf.base.shlepa/books_mc: Results for shlepa/books_mc:
INFO: 2024-11-26 20:27:39,209: llmtf.base.shlepa/books_mc: {'acc': 0.3112033195020747}
INFO: 2024-11-26 20:27:39,212: llmtf.base.evaluator: Ended eval
INFO: 2024-11-26 20:27:39,231: llmtf.base.evaluator:
mean NEREL daru/treewayabstractive darumeru/MultiQ darumeru/PARus darumeru/RCB darumeru/RWSD darumeru/USE darumeru/cp_para_ru ruopinionne ruparam shlepa/books_mc shlepa/law_mc shlepa/movie_mc shlepa/music_mc vikhrmodels/habr_qa_sbs
0.318 0.007 0.209 0.257 0.440 0.414 0.490 0.061 0.800 0.027 0.214 0.311 0.538 0.225 0.245 0.538
INFO: 2024-11-26 20:28:00,389: llmtf.base.nlpcoreteam/enMMLU: Processing Dataset: 487.59s
INFO: 2024-11-26 20:28:00,396: llmtf.base.nlpcoreteam/enMMLU: Results for nlpcoreteam/enMMLU:
INFO: 2024-11-26 20:28:00,445: llmtf.base.nlpcoreteam/enMMLU: metric
subject
abstract_algebra 0.310000
anatomy 0.518519
astronomy 0.717105
business_ethics 0.630000
clinical_knowledge 0.667925
college_biology 0.680556
college_chemistry 0.400000
college_computer_science 0.510000
college_mathematics 0.270000
college_medicine 0.618497
college_physics 0.558824
computer_security 0.750000
conceptual_physics 0.561702
econometrics 0.438596
electrical_engineering 0.579310
elementary_mathematics 0.455026
formal_logic 0.412698
global_facts 0.250000
high_school_biology 0.738710
high_school_chemistry 0.541872
high_school_computer_science 0.650000
high_school_european_history 0.751515
high_school_geography 0.767677
high_school_government_and_politics 0.808290
high_school_macroeconomics 0.638462
high_school_mathematics 0.433333
high_school_microeconomics 0.676471
high_school_physics 0.377483
high_school_psychology 0.814679
high_school_statistics 0.527778
high_school_us_history 0.715686
high_school_world_history 0.767932
human_aging 0.641256
human_sexuality 0.679389
international_law 0.735537
jurisprudence 0.777778
logical_fallacies 0.760736
machine_learning 0.455357
management 0.757282
marketing 0.837607
medical_genetics 0.690000
miscellaneous 0.713921
moral_disputes 0.650289
moral_scenarios 0.243575
nutrition 0.663399
philosophy 0.668810
prehistory 0.682099
professional_accounting 0.489362
professional_law 0.418514
professional_medicine 0.599265
professional_psychology 0.588235
public_relations 0.600000
security_studies 0.689796
sociology 0.786070
us_foreign_policy 0.780000
virology 0.463855
world_religions 0.789474
INFO: 2024-11-26 20:28:00,453: llmtf.base.nlpcoreteam/enMMLU: metric
subject
STEM 0.528725
humanities 0.644203
other (business, health, misc.) 0.610063
social sciences 0.688972
INFO: 2024-11-26 20:28:00,464: llmtf.base.nlpcoreteam/enMMLU: {'acc': 0.6179910195383496}
INFO: 2024-11-26 20:28:00,501: llmtf.base.evaluator: Ended eval
INFO: 2024-11-26 20:28:00,524: llmtf.base.evaluator:
mean NEREL daru/treewayabstractive darumeru/MultiQ darumeru/PARus darumeru/RCB darumeru/RWSD darumeru/USE darumeru/cp_para_ru nlpcoreteam/enMMLU ruopinionne ruparam shlepa/books_mc shlepa/law_mc shlepa/movie_mc shlepa/music_mc vikhrmodels/habr_qa_sbs
0.337 0.007 0.209 0.257 0.440 0.414 0.490 0.061 0.800 0.618 0.027 0.214 0.311 0.538 0.225 0.245 0.538
INFO: 2024-11-26 20:28:16,900: llmtf.base.nlpcoreteam/ruMMLU: Processing Dataset: 501.69s
INFO: 2024-11-26 20:28:16,902: llmtf.base.nlpcoreteam/ruMMLU: Results for nlpcoreteam/ruMMLU:
INFO: 2024-11-26 20:28:16,950: llmtf.base.nlpcoreteam/ruMMLU: metric
subject
abstract_algebra 0.320000
anatomy 0.414815
astronomy 0.572368
business_ethics 0.450000
clinical_knowledge 0.505660
college_biology 0.375000
college_chemistry 0.310000
college_computer_science 0.380000
college_mathematics 0.350000
college_medicine 0.508671
college_physics 0.431373
computer_security 0.530000
conceptual_physics 0.429787
econometrics 0.298246
electrical_engineering 0.448276
elementary_mathematics 0.417989
formal_logic 0.365079
global_facts 0.240000
high_school_biology 0.487097
high_school_chemistry 0.443350
high_school_computer_science 0.530000
high_school_european_history 0.654545
high_school_geography 0.525253
high_school_government_and_politics 0.481865
high_school_macroeconomics 0.430769
high_school_mathematics 0.392593
high_school_microeconomics 0.441176
high_school_physics 0.291391
high_school_psychology 0.572477
high_school_statistics 0.416667
high_school_us_history 0.495098
high_school_world_history 0.632911
human_aging 0.488789
human_sexuality 0.496183
international_law 0.677686
jurisprudence 0.574074
logical_fallacies 0.441718
machine_learning 0.321429
management 0.563107
marketing 0.722222
medical_genetics 0.470000
miscellaneous 0.536398
moral_disputes 0.517341
moral_scenarios 0.237989
nutrition 0.526144
philosophy 0.546624
prehistory 0.478395
professional_accounting 0.365248
professional_law 0.331160
professional_medicine 0.367647
professional_psychology 0.415033
public_relations 0.454545
security_studies 0.595918
sociology 0.631841
us_foreign_policy 0.710000
virology 0.409639
world_religions 0.549708
INFO: 2024-11-26 20:28:16,959: llmtf.base.nlpcoreteam/ruMMLU: metric
subject
STEM 0.413740
humanities 0.500179
other (business, health, misc.) 0.469167
social sciences 0.504442
INFO: 2024-11-26 20:28:16,967: llmtf.base.nlpcoreteam/ruMMLU: {'acc': 0.47188210698069283}
INFO: 2024-11-26 20:28:17,010: llmtf.base.evaluator: Ended eval
INFO: 2024-11-26 20:28:17,020: llmtf.base.evaluator:
mean NEREL daru/treewayabstractive darumeru/MultiQ darumeru/PARus darumeru/RCB darumeru/RWSD darumeru/USE darumeru/cp_para_ru nlpcoreteam/enMMLU nlpcoreteam/ruMMLU ruopinionne ruparam shlepa/books_mc shlepa/law_mc shlepa/movie_mc shlepa/music_mc vikhrmodels/habr_qa_sbs
0.345 0.007 0.209 0.257 0.440 0.414 0.490 0.061 0.800 0.618 0.472 0.027 0.214 0.311 0.538 0.225 0.245 0.538

2
llmtf_eval/evaluation_results.txt Normal file
View File

@@ -0,0 +1,2 @@
mean NEREL daru/treewayabstractive darumeru/MultiQ darumeru/PARus darumeru/RCB darumeru/RWSD darumeru/USE darumeru/cp_para_ru llm_as_judge nlpcoreteam/enMMLU nlpcoreteam/ruMMLU ruopinionne ruparam shlepa/books_mc shlepa/law_mc shlepa/movie_mc shlepa/music_mc vikhrmodels/habr_qa_sbs
0.349 0.007 0.209 0.257 0.440 0.414 0.490 0.061 0.800 0.420 0.618 0.472 0.027 0.214 0.311 0.538 0.225 0.245 0.538

41000
llmtf_eval/llm_as_judge.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

22
llmtf_eval/llm_as_judge_params.jsonl Normal file
View File

@@ -0,0 +1,22 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/workdir/models/Qwen2.5-72B-Instruct",
"api_base": "http://10.36.60.73:3040",
"generation_config": {
"max_new_tokens": 200,
"temperature": 0.0,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2"
},
"max_model_len": 32000
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 256,
"max_sample_per_dataset": 10000000000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/llm_as_judge_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "llm_as_judge",
"results": {
"score": 0.42
},
"leaderboard_result": 0.42
}

3
llmtf_eval/nlpcoreteam_enMMLU.jsonl Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d39638678b135a0ac575407993d8d413c727aa2d94d2de0aca2520744a2819ea
size 37122790

54
llmtf_eval/nlpcoreteam_enMMLU_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 64,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 10000000000000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/nlpcoreteam_enMMLU_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "nlpcoreteam/enMMLU",
"results": {
"acc": 0.6179910195383496
},
"leaderboard_result": 0.6179910195383496
}

3
llmtf_eval/nlpcoreteam_ruMMLU.jsonl Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c8a67a489efec02331bf36f2d61f632f8e982c340cfc1657e5ff3aa9ccc82849
size 43039512

54
llmtf_eval/nlpcoreteam_ruMMLU_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 64,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 10000000000000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/nlpcoreteam_ruMMLU_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "nlpcoreteam/ruMMLU",
"results": {
"acc": 0.47188210698069283
},
"leaderboard_result": 0.47188210698069283
}

17684
llmtf_eval/ruopinionne.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

54
llmtf_eval/ruopinionne_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 256,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 200,
"method": "generate"
}
}

7
llmtf_eval/ruopinionne_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "ruopinionne",
"results": {
"f1": 0.02701209922104298
},
"leaderboard_result": 0.02701209922104298
}

3
llmtf_eval/ruparam.jsonl Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d45cd9b4dd24149a1964646b1c3f49d2eae1fa1f73f313a0bb433aac98a22973
size 12306566

54
llmtf_eval/ruparam_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 1,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 5000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/ruparam_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "ruparam",
"results": {
"acc": 0.21363220494053065
},
"leaderboard_result": 0.21363220494053065
}

24100
llmtf_eval/shlepa_books_mc.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/shlepa_books_mc_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 1,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 5000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/shlepa_books_mc_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "shlepa/books_mc",
"results": {
"acc": 0.3112033195020747
},
"leaderboard_result": 0.3112033195020747
}

48550
llmtf_eval/shlepa_law_mc.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/shlepa_law_mc_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 1,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 5000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/shlepa_law_mc_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "shlepa/law_mc",
"results": {
"acc": 0.537590113285273
},
"leaderboard_result": 0.537590113285273
}

21600
llmtf_eval/shlepa_movie_mc.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/shlepa_movie_mc_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 1,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 5000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/shlepa_movie_mc_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "shlepa/movie_mc",
"results": {
"acc": 0.22453703703703703
},
"leaderboard_result": 0.22453703703703703
}

23500
llmtf_eval/shlepa_music_mc.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

54
llmtf_eval/shlepa_music_mc_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 1,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 5000,
"method": "calculate_tokens_proba"
}
}

7
llmtf_eval/shlepa_music_mc_total.jsonl Normal file
View File

@@ -0,0 +1,7 @@
{
"task_name": "shlepa/music_mc",
"results": {
"acc": 0.24468085106382978
},
"leaderboard_result": 0.24468085106382978
}

3
llmtf_eval/vikhrmodels_habr_qa_sbs.jsonl Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2ac8384f3c5fdf9c7157d765d17225068d7206e2e5bfe33569414a8f6f7dd82e
size 22011832

54
llmtf_eval/vikhrmodels_habr_qa_sbs_params.jsonl Normal file
View File

@@ -0,0 +1,54 @@
{
"custom_generation_config": null,
"model_params": {
"model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
"generation_config": {
"bos_token_id": 145109,
"do_sample": true,
"eos_token_id": [
145111
],
"max_length": 32768,
"max_new_tokens": 1,
"pad_token_id": 145109,
"stop_strings": [
"<|im_end|>"
],
"temperature": 0.1,
"top_k": 40,
"top_p": 0.9,
"transformers_version": "4.45.2",
"trust_remote_code": false
},
"conversation_template": {
"system_prompt": "",
"system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
"bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
"user_role": "user",
"bot_role": "assistant",
"system_role": "system",
"global_prefix": "",
"suffix": "<|im_start|>assistant\n",
"add_special_tokens": false,
"eos_token": "<|im_end|>"
},
"load_in_8bit": false,
"torch_dtype": "auto",
"attn_implementation": "flash_attention_2",
"device_map": "cuda:0",
"use_fast_tokenizer": true,
"leading_space": false,
"space_token": null,
"trust_remote_code": false,
"max_model_len": 32768
},
"task_params": {
"max_len": 4000,
"few_shot_count": 0,
"batch_size": 16,
"max_sample_per_dataset": 5000,
"method": "calculate_tokens_proba"
}
}

8
llmtf_eval/vikhrmodels_habr_qa_sbs_total.jsonl Normal file
View File

@@ -0,0 +1,8 @@
{
"task_name": "vikhrmodels/habr_qa_sbs",
"results": {
"acc": 0.547,
"f1_macro": 0.5280856541414141
},
"leaderboard_result": 0.537542827070707
}