evaluations: arc_challenge_poly_pt_acc: 0.37948717948717947 arc_challenge_poly_pt_acc_norm: 0.4282051282051282 arc_challenge_poly_pt_acc_norm_stderr: 0.014472341586181985 arc_challenge_poly_pt_acc_stderr: 0.014192754090886793 arc_challenge_poly_pt_alias: arc_challenge_poly_pt assin2_rte_acc,all: 0.8705065359477124 assin2_rte_acc_stderr,all: 0.004798540986291823 assin2_rte_alias: assin2_rte assin2_rte_f1_macro,all: 0.8691746625542952 assin2_rte_f1_macro_stderr,all: 0.004847278802663799 assin2_sts_alias: assin2_sts assin2_sts_mse,all: 2.2781209150326798 assin2_sts_mse_stderr,all: N/A assin2_sts_pearson,all: 0.47716809267162485 assin2_sts_pearson_stderr,all: 0.014407170423859306 assin_entailment_acc: 0.7325 assin_entailment_acc_stderr: 0.006999870502142285 assin_entailment_alias: assin_entailment assin_paraphrase_acc: 0.73625 assin_paraphrase_acc_stderr: 0.006968401827607802 assin_paraphrase_alias: assin_paraphrase belebele_por_Latn_acc: 0.6766666666666666 belebele_por_Latn_acc_norm: 0.6766666666666666 belebele_por_Latn_acc_norm_stderr: 0.015600294087844632 belebele_por_Latn_acc_stderr: 0.015600294087844632 belebele_por_Latn_alias: belebele_por_Latn bluex_acc,all: 0.39221140472878996 bluex_acc,exam_id__UNICAMP_2018: 0.48148148148148145 bluex_acc,exam_id__UNICAMP_2019: 0.42 bluex_acc,exam_id__UNICAMP_2020: 0.41818181818181815 bluex_acc,exam_id__UNICAMP_2021_1: 0.4782608695652174 bluex_acc,exam_id__UNICAMP_2021_2: 0.29411764705882354 bluex_acc,exam_id__UNICAMP_2022: 0.5384615384615384 bluex_acc,exam_id__UNICAMP_2023: 0.4883720930232558 bluex_acc,exam_id__UNICAMP_2024: 0.4444444444444444 bluex_acc,exam_id__USP_2018: 0.24074074074074073 bluex_acc,exam_id__USP_2019: 0.35 bluex_acc,exam_id__USP_2020: 0.39285714285714285 bluex_acc,exam_id__USP_2021: 0.3076923076923077 bluex_acc,exam_id__USP_2022: 0.24489795918367346 bluex_acc,exam_id__USP_2023: 0.38636363636363635 bluex_acc,exam_id__USP_2024: 0.4634146341463415 bluex_acc_stderr,all: 0.010527552369043944 bluex_acc_stderr,exam_id__UNICAMP_2018: 0.03932520147192548 bluex_acc_stderr,exam_id__UNICAMP_2019: 0.040409592319981036 bluex_acc_stderr,exam_id__UNICAMP_2020: 0.038500996629012005 bluex_acc_stderr,exam_id__UNICAMP_2021_1: 0.04265317780472471 bluex_acc_stderr,exam_id__UNICAMP_2021_2: 0.0369309745845929 bluex_acc_stderr,exam_id__UNICAMP_2022: 0.04616267465694302 bluex_acc_stderr,exam_id__UNICAMP_2023: 0.04391771882582978 bluex_acc_stderr,exam_id__UNICAMP_2024: 0.04266417007528182 bluex_acc_stderr,exam_id__USP_2018: 0.03361325629183327 bluex_acc_stderr,exam_id__USP_2019: 0.04344929186946837 bluex_acc_stderr,exam_id__USP_2020: 0.037457169801895486 bluex_acc_stderr,exam_id__USP_2021: 0.03696695228727266 bluex_acc_stderr,exam_id__USP_2022: 0.03545185428690165 bluex_acc_stderr,exam_id__USP_2023: 0.042474562735645566 bluex_acc_stderr,exam_id__USP_2024: 0.04499354148452882 bluex_alias: bluex calame_pt_acc: 0.11127167630057803 calame_pt_acc_stderr: 0.00690347530269077 calame_pt_alias: calame_pt calame_pt_perplexity: 5965.407764074068 calame_pt_perplexity_stderr: 683.2371253201818 enem_challenge_acc,all: 0.3988803358992302 enem_challenge_acc,exam_id__2009: 0.46956521739130436 enem_challenge_acc,exam_id__2010: 0.5128205128205128 enem_challenge_acc,exam_id__2011: 0.46153846153846156 enem_challenge_acc,exam_id__2012: 0.4224137931034483 enem_challenge_acc,exam_id__2013: 0.3148148148148148 enem_challenge_acc,exam_id__2014: 0.41284403669724773 enem_challenge_acc,exam_id__2015: 0.40336134453781514 enem_challenge_acc,exam_id__2016: 0.36363636363636365 enem_challenge_acc,exam_id__2016_2: 0.4065040650406504 enem_challenge_acc,exam_id__2017: 0.39655172413793105 enem_challenge_acc,exam_id__2022: 0.3157894736842105 enem_challenge_acc,exam_id__2023: 0.32592592592592595 enem_challenge_acc_stderr,all: 0.00749197527112301 enem_challenge_acc_stderr,exam_id__2009: 0.02676517541031027 enem_challenge_acc_stderr,exam_id__2010: 0.02676561702634192 enem_challenge_acc_stderr,exam_id__2011: 0.026575632846671127 enem_challenge_acc_stderr,exam_id__2012: 0.02652807977680181 enem_challenge_acc_stderr,exam_id__2013: 0.02570106484445073 enem_challenge_acc_stderr,exam_id__2014: 0.02722456002129014 enem_challenge_acc_stderr,exam_id__2015: 0.025882945505526844 enem_challenge_acc_stderr,exam_id__2016: 0.025343746567122242 enem_challenge_acc_stderr,exam_id__2016_2: 0.025491304109387235 enem_challenge_acc_stderr,exam_id__2017: 0.0261642629559971 enem_challenge_acc_stderr,exam_id__2022: 0.02331831926729694 enem_challenge_acc_stderr,exam_id__2023: 0.023372974631758865 enem_challenge_alias: enem faquad_nli_acc,all: 0.7892307692307692 faquad_nli_acc_stderr,all: 0.011279066985135663 faquad_nli_alias: faquad_nli faquad_nli_f1_macro,all: 0.49256657036543183 faquad_nli_f1_macro_stderr,all: 0.01318624622535848 global_piqa_completions_por_latn_braz_acc: 0.75 global_piqa_completions_por_latn_braz_acc_bytes: 0.74 global_piqa_completions_por_latn_braz_acc_bytes_stderr: 0.0440844002276808 global_piqa_completions_por_latn_braz_acc_norm: 0.74 global_piqa_completions_por_latn_braz_acc_norm_stderr: 0.0440844002276808 global_piqa_completions_por_latn_braz_acc_stderr: 0.04351941398892446 global_piqa_completions_por_latn_braz_alias: global_piqa_completions_por_latn_braz gsm8k_pt_alias: gsm8k_pt gsm8k_pt_exact_match,flexible-extract: 0.228310502283105 gsm8k_pt_exact_match,strict-match: 0.0 gsm8k_pt_exact_match_stderr,flexible-extract: 0.011583822031121497 gsm8k_pt_exact_match_stderr,strict-match: 0.0 hatebr_offensive_acc,all: 0.7071428571428572 hatebr_offensive_acc_stderr,all: 0.008611736471385361 hatebr_offensive_alias: hatebr_offensive_binary hatebr_offensive_f1_macro,all: 0.7019751844399732 hatebr_offensive_f1_macro_stderr,all: 0.008866946580220815 hellaswag_poly_pt_acc: 0.4334164048109221 hellaswag_poly_pt_acc_norm: 0.5494636471990465 hellaswag_poly_pt_acc_norm_stderr: 0.005179413791359488 hellaswag_poly_pt_acc_stderr: 0.005158588405358706 hellaswag_poly_pt_alias: hellaswag_poly_pt humaneval_instruct_alias: humaneval_instruct humaneval_instruct_pass@1,create_test: 0.0 humaneval_instruct_pass@1_stderr,create_test: 0.0 ifeval_pt_alias: ifeval_pt ifeval_pt_inst_level_loose_acc: 0.4511627906976744 ifeval_pt_inst_level_loose_acc_stderr: N/A ifeval_pt_inst_level_strict_acc: 0.3395348837209302 ifeval_pt_inst_level_strict_acc_stderr: N/A ifeval_pt_prompt_level_loose_acc: 0.33666666666666667 ifeval_pt_prompt_level_loose_acc_stderr: 0.02732941756218688 ifeval_pt_prompt_level_strict_acc: 0.23333333333333334 ifeval_pt_prompt_level_strict_acc_stderr: 0.024459979523511425 lambada_poly_pt_acc: 0.267028915195032 lambada_poly_pt_acc_stderr: 0.00616360274242743 lambada_poly_pt_alias: lambada_poly_pt lambada_poly_pt_perplexity: 214.12248648482947 lambada_poly_pt_perplexity_stderr: 13.012850955803207 mmlu_poly_pt_acc: 0.43297808465926146 mmlu_poly_pt_acc_stderr: 0.004292713120965234 mmlu_poly_pt_alias: mmlu_poly_pt oab_exams_acc,all: 0.3425968109339408 oab_exams_acc,exam_id__2010-01: 0.3176470588235294 oab_exams_acc,exam_id__2010-02: 0.33 oab_exams_acc,exam_id__2011-03: 0.3333333333333333 oab_exams_acc,exam_id__2011-04: 0.3625 oab_exams_acc,exam_id__2011-05: 0.325 oab_exams_acc,exam_id__2012-06: 0.35 oab_exams_acc,exam_id__2012-06a: 0.325 oab_exams_acc,exam_id__2012-07: 0.3375 oab_exams_acc,exam_id__2012-08: 0.325 oab_exams_acc,exam_id__2012-09: 0.24675324675324675 oab_exams_acc,exam_id__2013-10: 0.325 oab_exams_acc,exam_id__2013-11: 0.2875 oab_exams_acc,exam_id__2013-12: 0.3375 oab_exams_acc,exam_id__2014-13: 0.3875 oab_exams_acc,exam_id__2014-14: 0.325 oab_exams_acc,exam_id__2014-15: 0.41025641025641024 oab_exams_acc,exam_id__2015-16: 0.3125 oab_exams_acc,exam_id__2015-17: 0.3974358974358974 oab_exams_acc,exam_id__2015-18: 0.375 oab_exams_acc,exam_id__2016-19: 0.41025641025641024 oab_exams_acc,exam_id__2016-20: 0.4375 oab_exams_acc,exam_id__2016-20a: 0.325 oab_exams_acc,exam_id__2016-21: 0.25 oab_exams_acc,exam_id__2017-22: 0.375 oab_exams_acc,exam_id__2017-23: 0.3125 oab_exams_acc,exam_id__2017-24: 0.4 oab_exams_acc,exam_id__2018-25: 0.3375 oab_exams_acc_stderr,all: 0.005854656489923565 oab_exams_acc_stderr,exam_id__2010-01: 0.02913844109530932 oab_exams_acc_stderr,exam_id__2010-02: 0.02720413223389449 oab_exams_acc_stderr,exam_id__2011-03: 0.02739274623256926 oab_exams_acc_stderr,exam_id__2011-04: 0.03103610079124539 oab_exams_acc_stderr,exam_id__2011-05: 0.030180181541041882 oab_exams_acc_stderr,exam_id__2012-06: 0.030663426761582978 oab_exams_acc_stderr,exam_id__2012-06a: 0.03019542051107743 oab_exams_acc_stderr,exam_id__2012-07: 0.030587368127940558 oab_exams_acc_stderr,exam_id__2012-08: 0.030150394921254077 oab_exams_acc_stderr,exam_id__2012-09: 0.028367169852259254 oab_exams_acc_stderr,exam_id__2013-10: 0.030312606625205758 oab_exams_acc_stderr,exam_id__2013-11: 0.0291233179987808 oab_exams_acc_stderr,exam_id__2013-12: 0.030508999749351583 oab_exams_acc_stderr,exam_id__2014-13: 0.03131790740981766 oab_exams_acc_stderr,exam_id__2014-14: 0.030290399093110758 oab_exams_acc_stderr,exam_id__2014-15: 0.032059169494021815 oab_exams_acc_stderr,exam_id__2015-16: 0.029833251327948757 oab_exams_acc_stderr,exam_id__2015-17: 0.03201117621919014 oab_exams_acc_stderr,exam_id__2015-18: 0.0311783808239491 oab_exams_acc_stderr,exam_id__2016-19: 0.032162738371024235 oab_exams_acc_stderr,exam_id__2016-20: 0.031934266796334064 oab_exams_acc_stderr,exam_id__2016-20a: 0.030215012978448332 oab_exams_acc_stderr,exam_id__2016-21: 0.027979645441821365 oab_exams_acc_stderr,exam_id__2017-22: 0.031096010380049006 oab_exams_acc_stderr,exam_id__2017-23: 0.02995046656597778 oab_exams_acc_stderr,exam_id__2017-24: 0.03158083933896483 oab_exams_acc_stderr,exam_id__2018-25: 0.030481185333792096 oab_exams_alias: oab_exams portuguese_hate_speech_acc,all: 0.7097532314923619 portuguese_hate_speech_acc_stderr,all: 0.01097429657035187 portuguese_hate_speech_alias: portuguese_hate_speech_binary portuguese_hate_speech_f1_macro,all: 0.6922452504317789 portuguese_hate_speech_f1_macro_stderr,all: 0.011709050058396507 tweetsentbr_acc,all: 0.6422885572139303 tweetsentbr_acc_stderr,all: 0.007563312856378464 tweetsentbr_alias: tweetsentbr tweetsentbr_f1_macro,all: 0.6428335432993848 tweetsentbr_f1_macro_stderr,all: 0.007483756787178434 step: 3595