初始化项目，由ModelHub XC社区提供模型

Model: RefalMachine/RuadaptQwen2.5-1.5B-instruct
Source: Original Platform

.gitattributes

@@ -0,0 +1,41 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+llmtf_eval/daru_treewayextractive.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+llmtf_eval/ruparam.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval/vikhrmodels_habr_qa_sbs.jsonl filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,63 @@
+---
+datasets:
+- IlyaGusev/saiga_scored
+- IlyaGusev/saiga_preferences
+- dichspace/darulm
+language:
+- ru
+pipeline_tag: text-generation
+license: apache-2.0
+---
+
+## Описание модели
+
+WORK IN PROGRESS!!! Текущая версия v2.
+
+Инструктивная версия адаптированной на русский язык модели Qwen2.5-1.5B. В модели был заменен токенизатор, затем произведено дообучение (Continued pretraining) на русскоязычном корпусе, после чего была применена техника LEP (Learned Embedding Propagation, paper will be soon).
+
+Благодаря новому токенизатору (расширенный tiktoken cl100k с помощью униграм токенизатора на 48 т. токенов) скорость генерации* русскоязычных текстов возрасла до 60% по сравнению с исходной моделью Qwen-2.5-1.5B-Instruct.
+
+*Под скоростью генерации подразумевается количество русскоязычных символов/слов в секунду на одинаковых текстовых последовательностях.
+
+## Попробовать
+
+Модель можно попробовать в поднятом Space (внизу в параметрах выбор модели): 
+https://huggingface.co/spaces/RefalMachine/RuadaptQwen2.5
+
+## Токенизация
+
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/652cedbdf120598322ae358a/O4eQEhnowETEatDPcmArB.png)
+
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/652cedbdf120598322ae358a/oW0Q6LzD_Py3GdH0kfqu4.png)
+
+## Метрики и оценка качества
+
+Модель была оценена на Ru-Arena-General, MERA, llmtf_open
+
+#### Результаты на Ru-Arena-General
+
+Замеры были произведены с использованием оффициального кода лидерборда (https://github.com/VikhrModels/ru_llm_arena), **но с repetition_penalty=1.1**.
+
+TODO
+
+
+#### Результаты на MERA
+
+TODO
+
+
+#### Результаты на llmtf_open
+
+TODO
+
+## How to cite:
+
+Tikhomirov M., Chernyshev D. Facilitating large language model Russian adaptation with Learned Embedding Propagation // 2024 (Preprint: https://arxiv.org/abs/2412.21140)
+
+Tikhomirov M., Chernyshev D. Impact of Tokenization on LLaMa Russian Adaptation //2023 Ivannikov Ispras Open Conference (ISPRAS). – IEEE, 2023. – С. 163-168.
+
+## Предупреждение
+
+Ответы модели не отражают мнения авторов, а лишь повторяют знания полученные из данных на всех этапах обучения (предобучение, смена токенизатора, обучение на инструкциях, калибровка качества ответов). Модель была получена из сторонней предобученной модели, **контроль за предобучением** которой **не является ответственностью текущих авторов**. При создании данной версии модели не производилось никаких дополнительных действий, направленных на изменение заложенных в LLM "мнений". Используйте с осторожностью.

added_tokens.json

@@ -0,0 +1,45 @@
+{
+  "</tool_call>": 145124,
+  "<tool_call>": 145123,
+  "<|box_end|>": 145115,
+  "<|box_start|>": 145114,
+  "<|endoftext|>": 145109,
+  "<|file_sep|>": 145130,
+  "<|fim_middle|>": 145126,
+  "<|fim_pad|>": 145128,
+  "<|fim_prefix|>": 145125,
+  "<|fim_suffix|>": 145127,
+  "<|free_token10|>": 145140,
+  "<|free_token11|>": 145141,
+  "<|free_token12|>": 145142,
+  "<|free_token13|>": 145143,
+  "<|free_token14|>": 145144,
+  "<|free_token15|>": 145145,
+  "<|free_token16|>": 145146,
+  "<|free_token17|>": 145147,
+  "<|free_token18|>": 145148,
+  "<|free_token19|>": 145149,
+  "<|free_token1|>": 145131,
+  "<|free_token20|>": 145150,
+  "<|free_token21|>": 145151,
+  "<|free_token2|>": 145132,
+  "<|free_token3|>": 145133,
+  "<|free_token4|>": 145134,
+  "<|free_token5|>": 145135,
+  "<|free_token6|>": 145136,
+  "<|free_token7|>": 145137,
+  "<|free_token8|>": 145138,
+  "<|free_token9|>": 145139,
+  "<|im_end|>": 145111,
+  "<|im_start|>": 145110,
+  "<|image_pad|>": 145121,
+  "<|object_ref_end|>": 145113,
+  "<|object_ref_start|>": 145112,
+  "<|quad_end|>": 145117,
+  "<|quad_start|>": 145116,
+  "<|repo_name|>": 145129,
+  "<|video_pad|>": 145122,
+  "<|vision_end|>": 145119,
+  "<|vision_pad|>": 145120,
+  "<|vision_start|>": 145118
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/sft1",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "eos_token_id": 145111,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 145152
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "do_sample": true,
+  "eos_token_id": 145111,
+  "pad_token_id": 145109,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.45.2"
+}

llmtf_eval/NEREL_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 128,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 200,
+        "method": "generate"
+    }
+}

llmtf_eval/NEREL_total.jsonl

@@ -0,0 +1,10 @@
+{
+    "task_name": "NEREL",
+    "results": {
+        "tp": 2.0,
+        "fp": 27.0,
+        "fn": 519.0,
+        "micro-f1": 0.00727272727272595
+    },
+    "leaderboard_result": 0.00727272727272595
+}

llmtf_eval/daru_treewayabstractive_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 512,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 200,
+        "method": "generate"
+    }
+}

llmtf_eval/daru_treewayabstractive_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "daru/treewayabstractive",
+    "results": {
+        "rouge1": 0.3138417117532064,
+        "rouge2": 0.10462617373556911
+    },
+    "leaderboard_result": 0.20923394274438775
+}

llmtf_eval/daru_treewayextractive.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75cf70167322406bd8c1997ecfc9391b800f34de8cf75514b3d5394666e94263
+size 212316130

llmtf_eval/daru_treewayextractive_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 2,
+        "max_sample_per_dataset": 1000,
+        "method": "calculate_logsoftmax"
+    }
+}

llmtf_eval/daru_treewayextractive_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "daru/treewayextractive",
+    "results": {
+        "r-prec": 0.3782488455988456
+    },
+    "leaderboard_result": 0.3782488455988456
+}

llmtf_eval/darumeru_MultiQ_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval/darumeru_MultiQ_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/MultiQ",
+    "results": {
+        "f1": 0.3016876852043635,
+        "em": 0.21319311663479923
+    },
+    "leaderboard_result": 0.25744040091958137
+}

llmtf_eval/darumeru_PARus_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/darumeru_PARus_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/PARus",
+    "results": {
+        "acc": 0.44
+    },
+    "leaderboard_result": 0.44
+}

llmtf_eval/darumeru_RCB_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/darumeru_RCB_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/RCB",
+    "results": {
+        "acc": 0.4590909090909091,
+        "f1_macro": 0.36910715356478985
+    },
+    "leaderboard_result": 0.4140990313278495
+}

llmtf_eval/darumeru_RWSD_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/darumeru_RWSD_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/RWSD",
+    "results": {
+        "acc": 0.49019607843137253
+    },
+    "leaderboard_result": 0.49019607843137253
+}

llmtf_eval/darumeru_USE_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval/darumeru_USE_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/USE",
+    "results": {
+        "grade_norm": 0.06078431372549018
+    },
+    "leaderboard_result": 0.06078431372549018
+}

llmtf_eval/darumeru_cp_para_ru_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1024,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval/darumeru_cp_para_ru_total.jsonl

@@ -0,0 +1,10 @@
+{
+    "task_name": "darumeru/cp_para_ru",
+    "results": {
+        "tokens_per_word": 1.905314928817744,
+        "symbol_per_token": 3.913951487866651,
+        "len": 0.9904780330832407,
+        "lcs": 0.8
+    },
+    "leaderboard_result": 0.8
+}

llmtf_eval/darumeru_ruOpenBookQA_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 2,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/darumeru_ruOpenBookQA_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/ruOpenBookQA",
+    "results": {
+        "acc": 0.615979381443299,
+        "f1_macro": 0.6154023944317246
+    },
+    "leaderboard_result": 0.6156908879375118
+}

llmtf_eval/darumeru_ruWorldTree_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 2,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/darumeru_ruWorldTree_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/ruWorldTree",
+    "results": {
+        "acc": 0.7714285714285715,
+        "f1_macro": 0.7726851851851853
+    },
+    "leaderboard_result": 0.7720568783068784
+}

llmtf_eval/evaluation_log.txt

@@ -0,0 +1,272 @@
+INFO: 2024-11-26 20:17:25,790: llmtf.base.evaluator: Starting eval on ['darumeru/multiq', 'darumeru/parus', 'darumeru/rcb', 'darumeru/rwsd', 'darumeru/use']
+INFO: 2024-11-26 20:17:25,791: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:25,791: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:27,525: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/rummlu']
+INFO: 2024-11-26 20:17:27,525: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:27,525: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:29,517: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/enmmlu']
+INFO: 2024-11-26 20:17:29,517: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:29,517: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:30,060: llmtf.base.darumeru/MultiQ: Loading Dataset: 4.27s
+INFO: 2024-11-26 20:17:31,597: llmtf.base.evaluator: Starting eval on ['daru/treewayabstractive']
+INFO: 2024-11-26 20:17:31,597: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:31,597: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:33,345: llmtf.base.evaluator: Starting eval on ['darumeru/cp_para_ru']
+INFO: 2024-11-26 20:17:33,345: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:33,345: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:35,521: llmtf.base.evaluator: Starting eval on ['vikhrmodels/habr_qa_sbs', 'ruparam', 'shlepa/moviesmc', 'shlepa/musicmc', 'shlepa/lawmc', 'shlepa/booksmc']
+INFO: 2024-11-26 20:17:35,521: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:35,521: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:36,308: llmtf.base.darumeru/cp_para_ru: Loading Dataset: 2.96s
+INFO: 2024-11-26 20:17:36,742: llmtf.base.daru/treewayabstractive: Loading Dataset: 5.14s
+INFO: 2024-11-26 20:17:37,659: llmtf.base.evaluator: Starting eval on ['ruopinionne']
+INFO: 2024-11-26 20:17:37,659: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:37,660: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:37,984: llmtf.base.ruopinionne: Loading Dataset: 0.32s
+INFO: 2024-11-26 20:17:39,551: llmtf.base.evaluator: Starting eval on ['nerel']
+INFO: 2024-11-26 20:17:39,551: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:39,551: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:43,423: llmtf.base.NEREL: Loading Dataset: 3.87s
+INFO: 2024-11-26 20:17:48,557: llmtf.base.vikhrmodels/habr_qa_sbs: Loading Dataset: 13.04s
+INFO: 2024-11-26 20:18:44,135: llmtf.base.darumeru/cp_para_ru: Processing Dataset: 67.83s
+INFO: 2024-11-26 20:18:44,145: llmtf.base.darumeru/cp_para_ru: Results for darumeru/cp_para_ru:
+INFO: 2024-11-26 20:18:44,149: llmtf.base.darumeru/cp_para_ru: {'tokens_per_word': 1.905314928817744, 'symbol_per_token': 3.913951487866651, 'len': 0.9904780330832407, 'lcs': 0.8}
+INFO: 2024-11-26 20:18:44,150: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:18:44,152: llmtf.base.evaluator: 
+mean	darumeru/cp_para_ru
+0.800	0.800
+INFO: 2024-11-26 20:18:50,857: llmtf.base.NEREL: Processing Dataset: 67.43s
+INFO: 2024-11-26 20:18:50,860: llmtf.base.NEREL: Results for NEREL:
+INFO: 2024-11-26 20:18:50,864: llmtf.base.NEREL: {'tp': 2.0, 'fp': 27.0, 'fn': 519.0, 'micro-f1': 0.00727272727272595}
+INFO: 2024-11-26 20:18:50,865: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:18:50,869: llmtf.base.evaluator: 
+mean	NEREL	darumeru/cp_para_ru
+0.404	0.007	0.800
+INFO: 2024-11-26 20:18:57,627: llmtf.base.daru/treewayabstractive: Processing Dataset: 80.88s
+INFO: 2024-11-26 20:18:57,629: llmtf.base.daru/treewayabstractive: Results for daru/treewayabstractive:
+INFO: 2024-11-26 20:18:57,632: llmtf.base.daru/treewayabstractive: {'rouge1': 0.3138417117532064, 'rouge2': 0.10462617373556911}
+INFO: 2024-11-26 20:18:57,634: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:18:57,637: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/cp_para_ru
+0.339	0.007	0.209	0.800
+INFO: 2024-11-26 20:19:13,379: llmtf.base.darumeru/MultiQ: Processing Dataset: 103.32s
+INFO: 2024-11-26 20:19:13,384: llmtf.base.darumeru/MultiQ: Results for darumeru/MultiQ:
+INFO: 2024-11-26 20:19:13,404: llmtf.base.darumeru/MultiQ: {'f1': 0.3016876852043635, 'em': 0.21319311663479923}
+INFO: 2024-11-26 20:19:13,412: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:19:13,412: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:19:15,573: llmtf.base.darumeru/PARus: Loading Dataset: 2.16s
+INFO: 2024-11-26 20:19:19,861: llmtf.base.darumeru/PARus: Processing Dataset: 4.29s
+INFO: 2024-11-26 20:19:19,867: llmtf.base.darumeru/PARus: Results for darumeru/PARus:
+INFO: 2024-11-26 20:19:19,887: llmtf.base.darumeru/PARus: {'acc': 0.44}
+INFO: 2024-11-26 20:19:19,888: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:19:19,888: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:19:22,295: llmtf.base.darumeru/RCB: Loading Dataset: 2.40s
+INFO: 2024-11-26 20:19:23,604: llmtf.base.ruopinionne: Processing Dataset: 105.62s
+INFO: 2024-11-26 20:19:23,610: llmtf.base.ruopinionne: Results for ruopinionne:
+INFO: 2024-11-26 20:19:23,639: llmtf.base.ruopinionne: {'f1': 0.02701209922104298}
+INFO: 2024-11-26 20:19:23,640: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:19:23,656: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/cp_para_ru	ruopinionne
+0.290	0.007	0.209	0.257	0.440	0.800	0.027
+INFO: 2024-11-26 20:19:27,714: llmtf.base.darumeru/RCB: Processing Dataset: 5.42s
+INFO: 2024-11-26 20:19:27,715: llmtf.base.darumeru/RCB: Results for darumeru/RCB:
+INFO: 2024-11-26 20:19:27,722: llmtf.base.darumeru/RCB: {'acc': 0.4590909090909091, 'f1_macro': 0.36910715356478985}
+INFO: 2024-11-26 20:19:27,724: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:19:27,724: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:19:29,690: llmtf.base.darumeru/RWSD: Loading Dataset: 1.96s
+INFO: 2024-11-26 20:19:34,850: llmtf.base.darumeru/RWSD: Processing Dataset: 5.16s
+INFO: 2024-11-26 20:19:34,855: llmtf.base.darumeru/RWSD: Results for darumeru/RWSD:
+INFO: 2024-11-26 20:19:34,858: llmtf.base.darumeru/RWSD: {'acc': 0.49019607843137253}
+INFO: 2024-11-26 20:19:34,859: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:19:34,859: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:19:38,002: llmtf.base.darumeru/USE: Loading Dataset: 3.14s
+INFO: 2024-11-26 20:19:52,797: llmtf.base.nlpcoreteam/enMMLU: Loading Dataset: 143.28s
+INFO: 2024-11-26 20:19:55,208: llmtf.base.nlpcoreteam/ruMMLU: Loading Dataset: 147.68s
+INFO: 2024-11-26 20:20:33,495: llmtf.base.vikhrmodels/habr_qa_sbs: Processing Dataset: 164.94s
+INFO: 2024-11-26 20:20:33,496: llmtf.base.vikhrmodels/habr_qa_sbs: Results for vikhrmodels/habr_qa_sbs:
+INFO: 2024-11-26 20:20:33,533: llmtf.base.vikhrmodels/habr_qa_sbs: {'acc': 0.547, 'f1_macro': 0.5280856541414141}
+INFO: 2024-11-26 20:20:33,546: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:20:33,547: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:20:40,434: llmtf.base.ruparam: Loading Dataset: 6.89s
+INFO: 2024-11-26 20:21:07,899: llmtf.base.darumeru/USE: Processing Dataset: 89.90s
+INFO: 2024-11-26 20:21:07,900: llmtf.base.darumeru/USE: Results for darumeru/USE:
+INFO: 2024-11-26 20:21:07,942: llmtf.base.darumeru/USE: {'grade_norm': 0.06078431372549018}
+INFO: 2024-11-26 20:21:07,948: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:21:08,016: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	ruopinionne	vikhrmodels/habr_qa_sbs
+0.324	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.027	0.538
+INFO: 2024-11-26 20:25:43,772: llmtf.base.ruparam: Processing Dataset: 303.34s
+INFO: 2024-11-26 20:25:43,788: llmtf.base.ruparam: Results for ruparam:
+INFO: 2024-11-26 20:25:44,038: llmtf.base.ruparam: {'acc': 0.21363220494053065}
+INFO: 2024-11-26 20:25:44,053: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:25:44,053: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:25:47,874: llmtf.base.shlepa/movie_mc: Loading Dataset: 3.82s
+INFO: 2024-11-26 20:26:05,103: llmtf.base.shlepa/movie_mc: Processing Dataset: 17.22s
+INFO: 2024-11-26 20:26:05,119: llmtf.base.shlepa/movie_mc: Results for shlepa/movie_mc:
+INFO: 2024-11-26 20:26:05,122: llmtf.base.shlepa/movie_mc: {'acc': 0.22453703703703703}
+INFO: 2024-11-26 20:26:05,130: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:26:05,130: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:26:08,736: llmtf.base.shlepa/music_mc: Loading Dataset: 3.60s
+INFO: 2024-11-26 20:26:26,413: llmtf.base.shlepa/music_mc: Processing Dataset: 17.68s
+INFO: 2024-11-26 20:26:26,416: llmtf.base.shlepa/music_mc: Results for shlepa/music_mc:
+INFO: 2024-11-26 20:26:26,435: llmtf.base.shlepa/music_mc: {'acc': 0.24468085106382978}
+INFO: 2024-11-26 20:26:26,438: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:26:26,439: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:26:31,058: llmtf.base.shlepa/law_mc: Loading Dataset: 4.62s
+INFO: 2024-11-26 20:27:15,973: llmtf.base.shlepa/law_mc: Processing Dataset: 44.91s
+INFO: 2024-11-26 20:27:15,980: llmtf.base.shlepa/law_mc: Results for shlepa/law_mc:
+INFO: 2024-11-26 20:27:16,000: llmtf.base.shlepa/law_mc: {'acc': 0.537590113285273}
+INFO: 2024-11-26 20:27:16,006: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:27:16,006: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:27:19,668: llmtf.base.shlepa/books_mc: Loading Dataset: 3.66s
+INFO: 2024-11-26 20:27:39,200: llmtf.base.shlepa/books_mc: Processing Dataset: 19.53s
+INFO: 2024-11-26 20:27:39,204: llmtf.base.shlepa/books_mc: Results for shlepa/books_mc:
+INFO: 2024-11-26 20:27:39,209: llmtf.base.shlepa/books_mc: {'acc': 0.3112033195020747}
+INFO: 2024-11-26 20:27:39,212: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:27:39,231: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	ruopinionne	ruparam	shlepa/books_mc	shlepa/law_mc	shlepa/movie_mc	shlepa/music_mc	vikhrmodels/habr_qa_sbs
+0.318	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.027	0.214	0.311	0.538	0.225	0.245	0.538
+INFO: 2024-11-26 20:28:00,389: llmtf.base.nlpcoreteam/enMMLU: Processing Dataset: 487.59s
+INFO: 2024-11-26 20:28:00,396: llmtf.base.nlpcoreteam/enMMLU: Results for nlpcoreteam/enMMLU:
+INFO: 2024-11-26 20:28:00,445: llmtf.base.nlpcoreteam/enMMLU:                                        metric
+subject                                      
+abstract_algebra                     0.310000
+anatomy                              0.518519
+astronomy                            0.717105
+business_ethics                      0.630000
+clinical_knowledge                   0.667925
+college_biology                      0.680556
+college_chemistry                    0.400000
+college_computer_science             0.510000
+college_mathematics                  0.270000
+college_medicine                     0.618497
+college_physics                      0.558824
+computer_security                    0.750000
+conceptual_physics                   0.561702
+econometrics                         0.438596
+electrical_engineering               0.579310
+elementary_mathematics               0.455026
+formal_logic                         0.412698
+global_facts                         0.250000
+high_school_biology                  0.738710
+high_school_chemistry                0.541872
+high_school_computer_science         0.650000
+high_school_european_history         0.751515
+high_school_geography                0.767677
+high_school_government_and_politics  0.808290
+high_school_macroeconomics           0.638462
+high_school_mathematics              0.433333
+high_school_microeconomics           0.676471
+high_school_physics                  0.377483
+high_school_psychology               0.814679
+high_school_statistics               0.527778
+high_school_us_history               0.715686
+high_school_world_history            0.767932
+human_aging                          0.641256
+human_sexuality                      0.679389
+international_law                    0.735537
+jurisprudence                        0.777778
+logical_fallacies                    0.760736
+machine_learning                     0.455357
+management                           0.757282
+marketing                            0.837607
+medical_genetics                     0.690000
+miscellaneous                        0.713921
+moral_disputes                       0.650289
+moral_scenarios                      0.243575
+nutrition                            0.663399
+philosophy                           0.668810
+prehistory                           0.682099
+professional_accounting              0.489362
+professional_law                     0.418514
+professional_medicine                0.599265
+professional_psychology              0.588235
+public_relations                     0.600000
+security_studies                     0.689796
+sociology                            0.786070
+us_foreign_policy                    0.780000
+virology                             0.463855
+world_religions                      0.789474
+INFO: 2024-11-26 20:28:00,453: llmtf.base.nlpcoreteam/enMMLU:                                    metric
+subject                                  
+STEM                             0.528725
+humanities                       0.644203
+other (business, health, misc.)  0.610063
+social sciences                  0.688972
+INFO: 2024-11-26 20:28:00,464: llmtf.base.nlpcoreteam/enMMLU: {'acc': 0.6179910195383496}
+INFO: 2024-11-26 20:28:00,501: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:28:00,524: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	nlpcoreteam/enMMLU	ruopinionne	ruparam	shlepa/books_mc	shlepa/law_mc	shlepa/movie_mc	shlepa/music_mc	vikhrmodels/habr_qa_sbs
+0.337	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.618	0.027	0.214	0.311	0.538	0.225	0.245	0.538
+INFO: 2024-11-26 20:28:16,900: llmtf.base.nlpcoreteam/ruMMLU: Processing Dataset: 501.69s
+INFO: 2024-11-26 20:28:16,902: llmtf.base.nlpcoreteam/ruMMLU: Results for nlpcoreteam/ruMMLU:
+INFO: 2024-11-26 20:28:16,950: llmtf.base.nlpcoreteam/ruMMLU:                                        metric
+subject                                      
+abstract_algebra                     0.320000
+anatomy                              0.414815
+astronomy                            0.572368
+business_ethics                      0.450000
+clinical_knowledge                   0.505660
+college_biology                      0.375000
+college_chemistry                    0.310000
+college_computer_science             0.380000
+college_mathematics                  0.350000
+college_medicine                     0.508671
+college_physics                      0.431373
+computer_security                    0.530000
+conceptual_physics                   0.429787
+econometrics                         0.298246
+electrical_engineering               0.448276
+elementary_mathematics               0.417989
+formal_logic                         0.365079
+global_facts                         0.240000
+high_school_biology                  0.487097
+high_school_chemistry                0.443350
+high_school_computer_science         0.530000
+high_school_european_history         0.654545
+high_school_geography                0.525253
+high_school_government_and_politics  0.481865
+high_school_macroeconomics           0.430769
+high_school_mathematics              0.392593
+high_school_microeconomics           0.441176
+high_school_physics                  0.291391
+high_school_psychology               0.572477
+high_school_statistics               0.416667
+high_school_us_history               0.495098
+high_school_world_history            0.632911
+human_aging                          0.488789
+human_sexuality                      0.496183
+international_law                    0.677686
+jurisprudence                        0.574074
+logical_fallacies                    0.441718
+machine_learning                     0.321429
+management                           0.563107
+marketing                            0.722222
+medical_genetics                     0.470000
+miscellaneous                        0.536398
+moral_disputes                       0.517341
+moral_scenarios                      0.237989
+nutrition                            0.526144
+philosophy                           0.546624
+prehistory                           0.478395
+professional_accounting              0.365248
+professional_law                     0.331160
+professional_medicine                0.367647
+professional_psychology              0.415033
+public_relations                     0.454545
+security_studies                     0.595918
+sociology                            0.631841
+us_foreign_policy                    0.710000
+virology                             0.409639
+world_religions                      0.549708
+INFO: 2024-11-26 20:28:16,959: llmtf.base.nlpcoreteam/ruMMLU:                                    metric
+subject                                  
+STEM                             0.413740
+humanities                       0.500179
+other (business, health, misc.)  0.469167
+social sciences                  0.504442
+INFO: 2024-11-26 20:28:16,967: llmtf.base.nlpcoreteam/ruMMLU: {'acc': 0.47188210698069283}
+INFO: 2024-11-26 20:28:17,010: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:28:17,020: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	ruopinionne	ruparam	shlepa/books_mc	shlepa/law_mc	shlepa/movie_mc	shlepa/music_mc	vikhrmodels/habr_qa_sbs
+0.345	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.618	0.472	0.027	0.214	0.311	0.538	0.225	0.245	0.538

llmtf_eval/evaluation_results.txt

@@ -0,0 +1,2 @@
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	llm_as_judge	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	ruopinionne	ruparam	shlepa/books_mc	shlepa/law_mc	shlepa/movie_mc	shlepa/music_mc	vikhrmodels/habr_qa_sbs
+0.349	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.420	0.618	0.472	0.027	0.214	0.311	0.538	0.225	0.245	0.538

llmtf_eval/llm_as_judge_params.jsonl

@@ -0,0 +1,22 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/workdir/models/Qwen2.5-72B-Instruct",
+        "api_base": "http://10.36.60.73:3040",
+        "generation_config": {
+            "max_new_tokens": 200,
+            "temperature": 0.0,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2"
+        },
+        "max_model_len": 32000
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 256,
+        "max_sample_per_dataset": 10000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/llm_as_judge_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "llm_as_judge",
+    "results": {
+        "score": 0.42
+    },
+    "leaderboard_result": 0.42
+}

llmtf_eval/nlpcoreteam_enMMLU.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d39638678b135a0ac575407993d8d413c727aa2d94d2de0aca2520744a2819ea
+size 37122790

llmtf_eval/nlpcoreteam_enMMLU_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/nlpcoreteam_enMMLU_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "nlpcoreteam/enMMLU",
+    "results": {
+        "acc": 0.6179910195383496
+    },
+    "leaderboard_result": 0.6179910195383496
+}

llmtf_eval/nlpcoreteam_ruMMLU.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8a67a489efec02331bf36f2d61f632f8e982c340cfc1657e5ff3aa9ccc82849
+size 43039512

llmtf_eval/nlpcoreteam_ruMMLU_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/nlpcoreteam_ruMMLU_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "nlpcoreteam/ruMMLU",
+    "results": {
+        "acc": 0.47188210698069283
+    },
+    "leaderboard_result": 0.47188210698069283
+}

llmtf_eval/ruopinionne_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 256,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 200,
+        "method": "generate"
+    }
+}

llmtf_eval/ruopinionne_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "ruopinionne",
+    "results": {
+        "f1": 0.02701209922104298
+    },
+    "leaderboard_result": 0.02701209922104298
+}

llmtf_eval/ruparam.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d45cd9b4dd24149a1964646b1c3f49d2eae1fa1f73f313a0bb433aac98a22973
+size 12306566

llmtf_eval/ruparam_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 5000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/ruparam_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "ruparam",
+    "results": {
+        "acc": 0.21363220494053065
+    },
+    "leaderboard_result": 0.21363220494053065
+}

llmtf_eval/shlepa_books_mc_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 5000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/shlepa_books_mc_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "shlepa/books_mc",
+    "results": {
+        "acc": 0.3112033195020747
+    },
+    "leaderboard_result": 0.3112033195020747
+}

llmtf_eval/shlepa_law_mc_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 5000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/shlepa_law_mc_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "shlepa/law_mc",
+    "results": {
+        "acc": 0.537590113285273
+    },
+    "leaderboard_result": 0.537590113285273
+}

llmtf_eval/shlepa_movie_mc_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 5000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/shlepa_movie_mc_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "shlepa/movie_mc",
+    "results": {
+        "acc": 0.22453703703703703
+    },
+    "leaderboard_result": 0.22453703703703703
+}

llmtf_eval/shlepa_music_mc_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 5000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/shlepa_music_mc_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "shlepa/music_mc",
+    "results": {
+        "acc": 0.24468085106382978
+    },
+    "leaderboard_result": 0.24468085106382978
+}

llmtf_eval/vikhrmodels_habr_qa_sbs.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ac8384f3c5fdf9c7157d765d17225068d7206e2e5bfe33569414a8f6f7dd82e
+size 22011832

llmtf_eval/vikhrmodels_habr_qa_sbs_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 5000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/vikhrmodels_habr_qa_sbs_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "vikhrmodels/habr_qa_sbs",
+    "results": {
+        "acc": 0.547,
+        "f1_macro": 0.5280856541414141
+    },
+    "leaderboard_result": 0.537542827070707
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6e3e6a760fa3ced5dcf46192ec032d400f5106c3ed1498c1b1f3dbfaa992f9e
+size 3066626688

special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e03677792927c1ab6656bfc682845e327f47a996db955621143a6412b06c1908
+size 12298410

tokenizer_config.json

@@ -0,0 +1,375 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "145109": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145110": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145111": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145112": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145113": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145114": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145115": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145116": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145117": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145118": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145119": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145120": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145121": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145122": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145123": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145124": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145125": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145126": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145127": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145128": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145129": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145130": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145131": {
+      "content": "<|free_token1|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145132": {
+      "content": "<|free_token2|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145133": {
+      "content": "<|free_token3|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145134": {
+      "content": "<|free_token4|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145135": {
+      "content": "<|free_token5|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145136": {
+      "content": "<|free_token6|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145137": {
+      "content": "<|free_token7|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145138": {
+      "content": "<|free_token8|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145139": {
+      "content": "<|free_token9|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145140": {
+      "content": "<|free_token10|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145141": {
+      "content": "<|free_token11|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145142": {
+      "content": "<|free_token12|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145143": {
+      "content": "<|free_token13|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145144": {
+      "content": "<|free_token14|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145145": {
+      "content": "<|free_token15|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145146": {
+      "content": "<|free_token16|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145147": {
+      "content": "<|free_token17|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145148": {
+      "content": "<|free_token18|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145149": {
+      "content": "<|free_token19|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145150": {
+      "content": "<|free_token20|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145151": {
+      "content": "<|free_token21|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}