{ "results": { "openaimmlu": { " ": " ", "alias": "openaimmlu" }, "openaimmlu_STEM": { "acc,none": 0.4900662251655629, "acc_stderr,none": 0.00883192107765626, "alias": " - STEM" }, "openaimmlu_abstract_algebra": { "alias": " - abstract_algebra", "acc,none": 0.25, "acc_stderr,none": 0.04351941398892446 }, "openaimmlu_astronomy": { "alias": " - astronomy", "acc,none": 0.6842105263157895, "acc_stderr,none": 0.037827289808654685 }, "openaimmlu_college_biology": { "alias": " - college_biology", "acc,none": 0.6597222222222222, "acc_stderr,none": 0.039621355734862175 }, "openaimmlu_college_chemistry": { "alias": " - college_chemistry", "acc,none": 0.35, "acc_stderr,none": 0.047937248544110196 }, "openaimmlu_college_computer_science": { "alias": " - college_computer_science", "acc,none": 0.44, "acc_stderr,none": 0.04988876515698589 }, "openaimmlu_college_mathematics": { "alias": " - college_mathematics", "acc,none": 0.3, "acc_stderr,none": 0.046056618647183814 }, "openaimmlu_college_physics": { "alias": " - college_physics", "acc,none": 0.37254901960784315, "acc_stderr,none": 0.04810840148082633 }, "openaimmlu_computer_security": { "alias": " - computer_security", "acc,none": 0.71, "acc_stderr,none": 0.045604802157206845 }, "openaimmlu_conceptual_physics": { "alias": " - conceptual_physics", "acc,none": 0.548936170212766, "acc_stderr,none": 0.032529096196131965 }, "openaimmlu_econometrics": { "alias": " - econometrics", "acc,none": 0.3684210526315789, "acc_stderr,none": 0.04537815354939391 }, "openaimmlu_electrical_engineering": { "alias": " - electrical_engineering", "acc,none": 0.5103448275862069, "acc_stderr,none": 0.04165774775728763 }, "openaimmlu_elementary_mathematics": { "alias": " - elementary_mathematics", "acc,none": 0.48677248677248675, "acc_stderr,none": 0.025742297289575142 }, "openaimmlu_high_school_biology": { "alias": " - high_school_biology", "acc,none": 0.6645161290322581, "acc_stderr,none": 0.026860206444724352 }, "openaimmlu_high_school_chemistry": { "alias": " - high_school_chemistry", "acc,none": 0.4630541871921182, "acc_stderr,none": 0.035083705204426656 }, "openaimmlu_high_school_computer_science": { "alias": " - high_school_computer_science", "acc,none": 0.56, "acc_stderr,none": 0.04988876515698589 }, "openaimmlu_high_school_mathematics": { "alias": " - high_school_mathematics", "acc,none": 0.35185185185185186, "acc_stderr,none": 0.02911661760608301 }, "openaimmlu_high_school_physics": { "alias": " - high_school_physics", "acc,none": 0.37748344370860926, "acc_stderr,none": 0.039580272311215706 }, "openaimmlu_high_school_statistics": { "alias": " - high_school_statistics", "acc,none": 0.4675925925925926, "acc_stderr,none": 0.03402801581358966 }, "openaimmlu_humanities": { "acc,none": 0.6834811529933481, "acc_stderr,none": 0.01087157296938379, "alias": " - Humanities" }, "openaimmlu_high_school_european_history": { "alias": " - high_school_european_history", "acc,none": 0.7333333333333333, "acc_stderr,none": 0.03453131801885417 }, "openaimmlu_high_school_us_history": { "alias": " - high_school_us_history", "acc,none": 0.7254901960784313, "acc_stderr,none": 0.03132179803083291 }, "openaimmlu_high_school_world_history": { "alias": " - high_school_world_history", "acc,none": 0.7721518987341772, "acc_stderr,none": 0.027303484599069415 }, "openaimmlu_international_law": { "alias": " - international_law", "acc,none": 0.7355371900826446, "acc_stderr,none": 0.04026187527591205 }, "openaimmlu_jurisprudence": { "alias": " - jurisprudence", "acc,none": 0.6851851851851852, "acc_stderr,none": 0.04489931073591311 }, "openaimmlu_logical_fallacies": { "alias": " - logical_fallacies", "acc,none": 0.6871165644171779, "acc_stderr,none": 0.03642914578292404 }, "openaimmlu_philosophy": { "alias": " - philosophy", "acc,none": 0.6077170418006431, "acc_stderr,none": 0.027731258647011987 }, "openaimmlu_prehistory": { "alias": " - prehistory", "acc,none": 0.595679012345679, "acc_stderr,none": 0.027306625297327698 }, "openaimmlu_world_religions": { "alias": " - world_religions", "acc,none": 0.7251461988304093, "acc_stderr,none": 0.034240429246915824 }, "openaimmlu_other": { "acc,none": 0.5571476736345247, "acc_stderr,none": 0.0062200183711956835, "alias": " - Other" }, "openaimmlu_anatomy": { "alias": " - anatomy", "acc,none": 0.4740740740740741, "acc_stderr,none": 0.04313531696750575 }, "openaimmlu_clinical_knowledge": { "alias": " - clinical_knowledge", "acc,none": 0.5773584905660377, "acc_stderr,none": 0.030402331445769537 }, "openaimmlu_college_medicine": { "alias": " - college_medicine", "acc,none": 0.5086705202312138, "acc_stderr,none": 0.0381189098894041 }, "openaimmlu_formal_logic": { "alias": " - formal_logic", "acc,none": 0.3888888888888889, "acc_stderr,none": 0.04360314860077459 }, "openaimmlu_global_facts": { "alias": " - global_facts", "acc,none": 0.4, "acc_stderr,none": 0.049236596391733084 }, "openaimmlu_high_school_geography": { "alias": " - high_school_geography", "acc,none": 0.7121212121212122, "acc_stderr,none": 0.03225883512300992 }, "openaimmlu_high_school_psychology": { "alias": " - high_school_psychology", "acc,none": 0.7302752293577982, "acc_stderr,none": 0.01902848671111545 }, "openaimmlu_human_aging": { "alias": " - human_aging", "acc,none": 0.6278026905829597, "acc_stderr,none": 0.0324430528300873 }, "openaimmlu_machine_learning": { "alias": " - machine_learning", "acc,none": 0.41964285714285715, "acc_stderr,none": 0.04684099321077106 }, "openaimmlu_medical_genetics": { "alias": " - medical_genetics", "acc,none": 0.66, "acc_stderr,none": 0.04760952285695237 }, "openaimmlu_miscellaneous": { "alias": " - miscellaneous", "acc,none": 0.7573435504469987, "acc_stderr,none": 0.015329888940899873 }, "openaimmlu_nutrition": { "alias": " - nutrition", "acc,none": 0.6601307189542484, "acc_stderr,none": 0.027121956071388856 }, "openaimmlu_professional_accounting": { "alias": " - professional_accounting", "acc,none": 0.41843971631205673, "acc_stderr,none": 0.029427994039419994 }, "openaimmlu_professional_law": { "alias": " - professional_law", "acc,none": 0.41264667535853977, "acc_stderr,none": 0.012573836633799016 }, "openaimmlu_professional_medicine": { "alias": " - professional_medicine", "acc,none": 0.5735294117647058, "acc_stderr,none": 0.030042615832714857 }, "openaimmlu_professional_psychology": { "alias": " - professional_psychology", "acc,none": 0.5522875816993464, "acc_stderr,none": 0.020116925347422425 }, "openaimmlu_virology": { "alias": " - virology", "acc,none": 0.4759036144578313, "acc_stderr,none": 0.03887971849597264 }, "openaimmlu_social_science": { "acc,none": 0.5578210590383444, "acc_stderr,none": 0.008094265116110859, "alias": " - Social Science" }, "openaimmlu_business_ethics": { "alias": " - business_ethics", "acc,none": 0.67, "acc_stderr,none": 0.04725815626252609 }, "openaimmlu_high_school_government_and_politics": { "alias": " - high_school_government_and_politics", "acc,none": 0.772020725388601, "acc_stderr,none": 0.03027690994517826 }, "openaimmlu_high_school_macroeconomics": { "alias": " - high_school_macroeconomics", "acc,none": 0.5692307692307692, "acc_stderr,none": 0.025106820660539753 }, "openaimmlu_high_school_microeconomics": { "alias": " - high_school_microeconomics", "acc,none": 0.5756302521008403, "acc_stderr,none": 0.03210479051015776 }, "openaimmlu_human_sexuality": { "alias": " - human_sexuality", "acc,none": 0.6641221374045801, "acc_stderr,none": 0.04142313771996664 }, "openaimmlu_management": { "alias": " - management", "acc,none": 0.7281553398058253, "acc_stderr,none": 0.044052680241409216 }, "openaimmlu_marketing": { "alias": " - marketing", "acc,none": 0.8076923076923077, "acc_stderr,none": 0.025819233256483727 }, "openaimmlu_moral_disputes": { "alias": " - moral_disputes", "acc,none": 0.5751445086705202, "acc_stderr,none": 0.026613350840261746 }, "openaimmlu_moral_scenarios": { "alias": " - moral_scenarios", "acc,none": 0.2916201117318436, "acc_stderr,none": 0.015201032512520442 }, "openaimmlu_public_relations": { "alias": " - public_relations", "acc,none": 0.5727272727272728, "acc_stderr,none": 0.047381987035454834 }, "openaimmlu_security_studies": { "alias": " - security_studies", "acc,none": 0.6693877551020408, "acc_stderr,none": 0.030116426296540603 }, "openaimmlu_sociology": { "alias": " - sociology", "acc,none": 0.6915422885572139, "acc_stderr,none": 0.032658195885126966 }, "openaimmlu_us_foreign_policy": { "alias": " - us_foreign_policy", "acc,none": 0.81, "acc_stderr,none": 0.039427724440366234 } }, "groups": { "openaimmlu_STEM": { "acc,none": 0.4900662251655629, "acc_stderr,none": 0.00883192107765626, "alias": " - STEM" }, "openaimmlu_humanities": { "acc,none": 0.6834811529933481, "acc_stderr,none": 0.01087157296938379, "alias": " - Humanities" }, "openaimmlu_other": { "acc,none": 0.5571476736345247, "acc_stderr,none": 0.0062200183711956835, "alias": " - Other" }, "openaimmlu_social_science": { "acc,none": 0.5578210590383444, "acc_stderr,none": 0.008094265116110859, "alias": " - Social Science" } }, "group_subtasks": { "openaimmlu_humanities": [ "openaimmlu_jurisprudence", "openaimmlu_logical_fallacies", "openaimmlu_philosophy", "openaimmlu_high_school_world_history", "openaimmlu_high_school_european_history", "openaimmlu_prehistory", "openaimmlu_high_school_us_history", "openaimmlu_international_law", "openaimmlu_world_religions" ], "openaimmlu_social_science": [ "openaimmlu_high_school_microeconomics", "openaimmlu_high_school_government_and_politics", "openaimmlu_management", "openaimmlu_security_studies", "openaimmlu_business_ethics", "openaimmlu_sociology", "openaimmlu_high_school_macroeconomics", "openaimmlu_moral_scenarios", "openaimmlu_public_relations", "openaimmlu_us_foreign_policy", "openaimmlu_moral_disputes", "openaimmlu_human_sexuality", "openaimmlu_marketing" ], "openaimmlu_other": [ "openaimmlu_nutrition", "openaimmlu_miscellaneous", "openaimmlu_anatomy", "openaimmlu_virology", "openaimmlu_professional_medicine", "openaimmlu_human_aging", "openaimmlu_clinical_knowledge", "openaimmlu_professional_accounting", "openaimmlu_high_school_geography", "openaimmlu_professional_psychology", "openaimmlu_high_school_psychology", "openaimmlu_machine_learning", "openaimmlu_medical_genetics", "openaimmlu_professional_law", "openaimmlu_college_medicine", "openaimmlu_formal_logic", "openaimmlu_global_facts" ], "openaimmlu_STEM": [ "openaimmlu_college_physics", "openaimmlu_astronomy", "openaimmlu_computer_security", "openaimmlu_elementary_mathematics", "openaimmlu_high_school_chemistry", "openaimmlu_college_mathematics", "openaimmlu_college_chemistry", "openaimmlu_college_biology", "openaimmlu_conceptual_physics", "openaimmlu_high_school_statistics", "openaimmlu_electrical_engineering", "openaimmlu_high_school_computer_science", "openaimmlu_high_school_mathematics", "openaimmlu_abstract_algebra", "openaimmlu_high_school_physics", "openaimmlu_college_computer_science", "openaimmlu_econometrics", "openaimmlu_high_school_biology" ], "openaimmlu": [ "openaimmlu_STEM", "openaimmlu_other", "openaimmlu_social_science", "openaimmlu_humanities" ] }, "configs": { "openaimmlu_abstract_algebra": { "task": "openaimmlu_abstract_algebra", "task_alias": "abstract_algebra", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "abstract_algebra", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_anatomy": { "task": "openaimmlu_anatomy", "task_alias": "anatomy", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "anatomy", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_astronomy": { "task": "openaimmlu_astronomy", "task_alias": "astronomy", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "astronomy", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_business_ethics": { "task": "openaimmlu_business_ethics", "task_alias": "business_ethics", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "business_ethics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_clinical_knowledge": { "task": "openaimmlu_clinical_knowledge", "task_alias": "clinical_knowledge", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "clinical_knowledge", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_college_biology": { "task": "openaimmlu_college_biology", "task_alias": "college_biology", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "college_biology", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_college_chemistry": { "task": "openaimmlu_college_chemistry", "task_alias": "college_chemistry", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "college_chemistry", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_college_computer_science": { "task": "openaimmlu_college_computer_science", "task_alias": "college_computer_science", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "college_computer_science", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_college_mathematics": { "task": "openaimmlu_college_mathematics", "task_alias": "college_mathematics", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "college_mathematics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_college_medicine": { "task": "openaimmlu_college_medicine", "task_alias": "college_medicine", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "college_medicine", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_college_physics": { "task": "openaimmlu_college_physics", "task_alias": "college_physics", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "college_physics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_computer_security": { "task": "openaimmlu_computer_security", "task_alias": "computer_security", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "computer_security", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_conceptual_physics": { "task": "openaimmlu_conceptual_physics", "task_alias": "conceptual_physics", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "conceptual_physics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_econometrics": { "task": "openaimmlu_econometrics", "task_alias": "econometrics", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "econometrics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_electrical_engineering": { "task": "openaimmlu_electrical_engineering", "task_alias": "electrical_engineering", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "electrical_engineering", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_elementary_mathematics": { "task": "openaimmlu_elementary_mathematics", "task_alias": "elementary_mathematics", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "elementary_mathematics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_formal_logic": { "task": "openaimmlu_formal_logic", "task_alias": "formal_logic", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "formal_logic", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_global_facts": { "task": "openaimmlu_global_facts", "task_alias": "global_facts", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "global_facts", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_biology": { "task": "openaimmlu_high_school_biology", "task_alias": "high_school_biology", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_biology", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_chemistry": { "task": "openaimmlu_high_school_chemistry", "task_alias": "high_school_chemistry", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_chemistry", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_computer_science": { "task": "openaimmlu_high_school_computer_science", "task_alias": "high_school_computer_science", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_computer_science", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_european_history": { "task": "openaimmlu_high_school_european_history", "task_alias": "high_school_european_history", "tag": "openaimmlu_humanities_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_european_history", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_geography": { "task": "openaimmlu_high_school_geography", "task_alias": "high_school_geography", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_geography", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_government_and_politics": { "task": "openaimmlu_high_school_government_and_politics", "task_alias": "high_school_government_and_politics", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_government_and_politics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_macroeconomics": { "task": "openaimmlu_high_school_macroeconomics", "task_alias": "high_school_macroeconomics", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_macroeconomics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_mathematics": { "task": "openaimmlu_high_school_mathematics", "task_alias": "high_school_mathematics", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_mathematics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_microeconomics": { "task": "openaimmlu_high_school_microeconomics", "task_alias": "high_school_microeconomics", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_microeconomics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_physics": { "task": "openaimmlu_high_school_physics", "task_alias": "high_school_physics", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_physics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_psychology": { "task": "openaimmlu_high_school_psychology", "task_alias": "high_school_psychology", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_psychology", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_statistics": { "task": "openaimmlu_high_school_statistics", "task_alias": "high_school_statistics", "tag": "openaimmlu_STEM_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_statistics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_us_history": { "task": "openaimmlu_high_school_us_history", "task_alias": "high_school_us_history", "tag": "openaimmlu_humanities_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_us_history", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_high_school_world_history": { "task": "openaimmlu_high_school_world_history", "task_alias": "high_school_world_history", "tag": "openaimmlu_humanities_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "high_school_world_history", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_human_aging": { "task": "openaimmlu_human_aging", "task_alias": "human_aging", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "human_aging", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_human_sexuality": { "task": "openaimmlu_human_sexuality", "task_alias": "human_sexuality", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "human_sexuality", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_international_law": { "task": "openaimmlu_international_law", "task_alias": "international_law", "tag": "openaimmlu_humanities_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "international_law", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_jurisprudence": { "task": "openaimmlu_jurisprudence", "task_alias": "jurisprudence", "tag": "openaimmlu_humanities_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "jurisprudence", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_logical_fallacies": { "task": "openaimmlu_logical_fallacies", "task_alias": "logical_fallacies", "tag": "openaimmlu_humanities_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "logical_fallacies", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_machine_learning": { "task": "openaimmlu_machine_learning", "task_alias": "machine_learning", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "machine_learning", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_management": { "task": "openaimmlu_management", "task_alias": "management", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "management", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_marketing": { "task": "openaimmlu_marketing", "task_alias": "marketing", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "marketing", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_medical_genetics": { "task": "openaimmlu_medical_genetics", "task_alias": "medical_genetics", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "medical_genetics", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_miscellaneous": { "task": "openaimmlu_miscellaneous", "task_alias": "miscellaneous", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "miscellaneous", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_moral_disputes": { "task": "openaimmlu_moral_disputes", "task_alias": "moral_disputes", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "moral_disputes", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_moral_scenarios": { "task": "openaimmlu_moral_scenarios", "task_alias": "moral_scenarios", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "moral_scenarios", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_nutrition": { "task": "openaimmlu_nutrition", "task_alias": "nutrition", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "nutrition", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_philosophy": { "task": "openaimmlu_philosophy", "task_alias": "philosophy", "tag": "openaimmlu_humanities_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "philosophy", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_prehistory": { "task": "openaimmlu_prehistory", "task_alias": "prehistory", "tag": "openaimmlu_humanities_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "prehistory", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_professional_accounting": { "task": "openaimmlu_professional_accounting", "task_alias": "professional_accounting", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "professional_accounting", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_professional_law": { "task": "openaimmlu_professional_law", "task_alias": "professional_law", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "professional_law", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_professional_medicine": { "task": "openaimmlu_professional_medicine", "task_alias": "professional_medicine", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "professional_medicine", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_professional_psychology": { "task": "openaimmlu_professional_psychology", "task_alias": "professional_psychology", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "professional_psychology", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_public_relations": { "task": "openaimmlu_public_relations", "task_alias": "public_relations", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "public_relations", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_security_studies": { "task": "openaimmlu_security_studies", "task_alias": "security_studies", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "security_studies", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_sociology": { "task": "openaimmlu_sociology", "task_alias": "sociology", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "sociology", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_us_foreign_policy": { "task": "openaimmlu_us_foreign_policy", "task_alias": "us_foreign_policy", "tag": "openaimmlu_social_science_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "us_foreign_policy", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_virology": { "task": "openaimmlu_virology", "task_alias": "virology", "tag": "openaimmlu_other_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "virology", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } }, "openaimmlu_world_religions": { "task": "openaimmlu_world_religions", "task_alias": "world_religions", "tag": "openaimmlu_humanities_tasks", "dataset_path": "khalidalt/openai_mmlu_arabic", "dataset_name": "world_religions", "test_split": "test", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", "doc_to_text": "query", "doc_to_target": "gold", "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } } }, "versions": { "openaimmlu_STEM": 0, "openaimmlu_abstract_algebra": 0.0, "openaimmlu_anatomy": 0.0, "openaimmlu_astronomy": 0.0, "openaimmlu_business_ethics": 0.0, "openaimmlu_clinical_knowledge": 0.0, "openaimmlu_college_biology": 0.0, "openaimmlu_college_chemistry": 0.0, "openaimmlu_college_computer_science": 0.0, "openaimmlu_college_mathematics": 0.0, "openaimmlu_college_medicine": 0.0, "openaimmlu_college_physics": 0.0, "openaimmlu_computer_security": 0.0, "openaimmlu_conceptual_physics": 0.0, "openaimmlu_econometrics": 0.0, "openaimmlu_electrical_engineering": 0.0, "openaimmlu_elementary_mathematics": 0.0, "openaimmlu_formal_logic": 0.0, "openaimmlu_global_facts": 0.0, "openaimmlu_high_school_biology": 0.0, "openaimmlu_high_school_chemistry": 0.0, "openaimmlu_high_school_computer_science": 0.0, "openaimmlu_high_school_european_history": 0.0, "openaimmlu_high_school_geography": 0.0, "openaimmlu_high_school_government_and_politics": 0.0, "openaimmlu_high_school_macroeconomics": 0.0, "openaimmlu_high_school_mathematics": 0.0, "openaimmlu_high_school_microeconomics": 0.0, "openaimmlu_high_school_physics": 0.0, "openaimmlu_high_school_psychology": 0.0, "openaimmlu_high_school_statistics": 0.0, "openaimmlu_high_school_us_history": 0.0, "openaimmlu_high_school_world_history": 0.0, "openaimmlu_human_aging": 0.0, "openaimmlu_human_sexuality": 0.0, "openaimmlu_humanities": 0, "openaimmlu_international_law": 0.0, "openaimmlu_jurisprudence": 0.0, "openaimmlu_logical_fallacies": 0.0, "openaimmlu_machine_learning": 0.0, "openaimmlu_management": 0.0, "openaimmlu_marketing": 0.0, "openaimmlu_medical_genetics": 0.0, "openaimmlu_miscellaneous": 0.0, "openaimmlu_moral_disputes": 0.0, "openaimmlu_moral_scenarios": 0.0, "openaimmlu_nutrition": 0.0, "openaimmlu_other": 0, "openaimmlu_philosophy": 0.0, "openaimmlu_prehistory": 0.0, "openaimmlu_professional_accounting": 0.0, "openaimmlu_professional_law": 0.0, "openaimmlu_professional_medicine": 0.0, "openaimmlu_professional_psychology": 0.0, "openaimmlu_public_relations": 0.0, "openaimmlu_security_studies": 0.0, "openaimmlu_social_science": 0, "openaimmlu_sociology": 0.0, "openaimmlu_us_foreign_policy": 0.0, "openaimmlu_virology": 0.0, "openaimmlu_world_religions": 0.0 }, "n-shot": { "openaimmlu_abstract_algebra": 0, "openaimmlu_anatomy": 0, "openaimmlu_astronomy": 0, "openaimmlu_business_ethics": 0, "openaimmlu_clinical_knowledge": 0, "openaimmlu_college_biology": 0, "openaimmlu_college_chemistry": 0, "openaimmlu_college_computer_science": 0, "openaimmlu_college_mathematics": 0, "openaimmlu_college_medicine": 0, "openaimmlu_college_physics": 0, "openaimmlu_computer_security": 0, "openaimmlu_conceptual_physics": 0, "openaimmlu_econometrics": 0, "openaimmlu_electrical_engineering": 0, "openaimmlu_elementary_mathematics": 0, "openaimmlu_formal_logic": 0, "openaimmlu_global_facts": 0, "openaimmlu_high_school_biology": 0, "openaimmlu_high_school_chemistry": 0, "openaimmlu_high_school_computer_science": 0, "openaimmlu_high_school_european_history": 0, "openaimmlu_high_school_geography": 0, "openaimmlu_high_school_government_and_politics": 0, "openaimmlu_high_school_macroeconomics": 0, "openaimmlu_high_school_mathematics": 0, "openaimmlu_high_school_microeconomics": 0, "openaimmlu_high_school_physics": 0, "openaimmlu_high_school_psychology": 0, "openaimmlu_high_school_statistics": 0, "openaimmlu_high_school_us_history": 0, "openaimmlu_high_school_world_history": 0, "openaimmlu_human_aging": 0, "openaimmlu_human_sexuality": 0, "openaimmlu_international_law": 0, "openaimmlu_jurisprudence": 0, "openaimmlu_logical_fallacies": 0, "openaimmlu_machine_learning": 0, "openaimmlu_management": 0, "openaimmlu_marketing": 0, "openaimmlu_medical_genetics": 0, "openaimmlu_miscellaneous": 0, "openaimmlu_moral_disputes": 0, "openaimmlu_moral_scenarios": 0, "openaimmlu_nutrition": 0, "openaimmlu_philosophy": 0, "openaimmlu_prehistory": 0, "openaimmlu_professional_accounting": 0, "openaimmlu_professional_law": 0, "openaimmlu_professional_medicine": 0, "openaimmlu_professional_psychology": 0, "openaimmlu_public_relations": 0, "openaimmlu_security_studies": 0, "openaimmlu_sociology": 0, "openaimmlu_us_foreign_policy": 0, "openaimmlu_virology": 0, "openaimmlu_world_religions": 0 }, "higher_is_better": { "openaimmlu": { "acc": true }, "openaimmlu_STEM": { "acc": true }, "openaimmlu_abstract_algebra": { "acc": true }, "openaimmlu_anatomy": { "acc": true }, "openaimmlu_astronomy": { "acc": true }, "openaimmlu_business_ethics": { "acc": true }, "openaimmlu_clinical_knowledge": { "acc": true }, "openaimmlu_college_biology": { "acc": true }, "openaimmlu_college_chemistry": { "acc": true }, "openaimmlu_college_computer_science": { "acc": true }, "openaimmlu_college_mathematics": { "acc": true }, "openaimmlu_college_medicine": { "acc": true }, "openaimmlu_college_physics": { "acc": true }, "openaimmlu_computer_security": { "acc": true }, "openaimmlu_conceptual_physics": { "acc": true }, "openaimmlu_econometrics": { "acc": true }, "openaimmlu_electrical_engineering": { "acc": true }, "openaimmlu_elementary_mathematics": { "acc": true }, "openaimmlu_formal_logic": { "acc": true }, "openaimmlu_global_facts": { "acc": true }, "openaimmlu_high_school_biology": { "acc": true }, "openaimmlu_high_school_chemistry": { "acc": true }, "openaimmlu_high_school_computer_science": { "acc": true }, "openaimmlu_high_school_european_history": { "acc": true }, "openaimmlu_high_school_geography": { "acc": true }, "openaimmlu_high_school_government_and_politics": { "acc": true }, "openaimmlu_high_school_macroeconomics": { "acc": true }, "openaimmlu_high_school_mathematics": { "acc": true }, "openaimmlu_high_school_microeconomics": { "acc": true }, "openaimmlu_high_school_physics": { "acc": true }, "openaimmlu_high_school_psychology": { "acc": true }, "openaimmlu_high_school_statistics": { "acc": true }, "openaimmlu_high_school_us_history": { "acc": true }, "openaimmlu_high_school_world_history": { "acc": true }, "openaimmlu_human_aging": { "acc": true }, "openaimmlu_human_sexuality": { "acc": true }, "openaimmlu_humanities": { "acc": true }, "openaimmlu_international_law": { "acc": true }, "openaimmlu_jurisprudence": { "acc": true }, "openaimmlu_logical_fallacies": { "acc": true }, "openaimmlu_machine_learning": { "acc": true }, "openaimmlu_management": { "acc": true }, "openaimmlu_marketing": { "acc": true }, "openaimmlu_medical_genetics": { "acc": true }, "openaimmlu_miscellaneous": { "acc": true }, "openaimmlu_moral_disputes": { "acc": true }, "openaimmlu_moral_scenarios": { "acc": true }, "openaimmlu_nutrition": { "acc": true }, "openaimmlu_other": { "acc": true }, "openaimmlu_philosophy": { "acc": true }, "openaimmlu_prehistory": { "acc": true }, "openaimmlu_professional_accounting": { "acc": true }, "openaimmlu_professional_law": { "acc": true }, "openaimmlu_professional_medicine": { "acc": true }, "openaimmlu_professional_psychology": { "acc": true }, "openaimmlu_public_relations": { "acc": true }, "openaimmlu_security_studies": { "acc": true }, "openaimmlu_social_science": { "acc": true }, "openaimmlu_sociology": { "acc": true }, "openaimmlu_us_foreign_policy": { "acc": true }, "openaimmlu_virology": { "acc": true }, "openaimmlu_world_religions": { "acc": true } }, "n-samples": { "openaimmlu_college_physics": { "original": 102, "effective": 102 }, "openaimmlu_astronomy": { "original": 152, "effective": 152 }, "openaimmlu_computer_security": { "original": 100, "effective": 100 }, "openaimmlu_elementary_mathematics": { "original": 378, "effective": 378 }, "openaimmlu_high_school_chemistry": { "original": 203, "effective": 203 }, "openaimmlu_college_mathematics": { "original": 100, "effective": 100 }, "openaimmlu_college_chemistry": { "original": 100, "effective": 100 }, "openaimmlu_college_biology": { "original": 144, "effective": 144 }, "openaimmlu_conceptual_physics": { "original": 235, "effective": 235 }, "openaimmlu_high_school_statistics": { "original": 216, "effective": 216 }, "openaimmlu_electrical_engineering": { "original": 145, "effective": 145 }, "openaimmlu_high_school_computer_science": { "original": 100, "effective": 100 }, "openaimmlu_high_school_mathematics": { "original": 270, "effective": 270 }, "openaimmlu_abstract_algebra": { "original": 100, "effective": 100 }, "openaimmlu_high_school_physics": { "original": 151, "effective": 151 }, "openaimmlu_college_computer_science": { "original": 100, "effective": 100 }, "openaimmlu_econometrics": { "original": 114, "effective": 114 }, "openaimmlu_high_school_biology": { "original": 310, "effective": 310 }, "openaimmlu_nutrition": { "original": 306, "effective": 306 }, "openaimmlu_miscellaneous": { "original": 783, "effective": 783 }, "openaimmlu_anatomy": { "original": 135, "effective": 135 }, "openaimmlu_virology": { "original": 166, "effective": 166 }, "openaimmlu_professional_medicine": { "original": 272, "effective": 272 }, "openaimmlu_human_aging": { "original": 223, "effective": 223 }, "openaimmlu_clinical_knowledge": { "original": 265, "effective": 265 }, "openaimmlu_professional_accounting": { "original": 282, "effective": 282 }, "openaimmlu_high_school_geography": { "original": 198, "effective": 198 }, "openaimmlu_professional_psychology": { "original": 612, "effective": 612 }, "openaimmlu_high_school_psychology": { "original": 545, "effective": 545 }, "openaimmlu_machine_learning": { "original": 112, "effective": 112 }, "openaimmlu_medical_genetics": { "original": 100, "effective": 100 }, "openaimmlu_professional_law": { "original": 1534, "effective": 1534 }, "openaimmlu_college_medicine": { "original": 173, "effective": 173 }, "openaimmlu_formal_logic": { "original": 126, "effective": 126 }, "openaimmlu_global_facts": { "original": 100, "effective": 100 }, "openaimmlu_high_school_microeconomics": { "original": 238, "effective": 238 }, "openaimmlu_high_school_government_and_politics": { "original": 193, "effective": 193 }, "openaimmlu_management": { "original": 103, "effective": 103 }, "openaimmlu_security_studies": { "original": 245, "effective": 245 }, "openaimmlu_business_ethics": { "original": 100, "effective": 100 }, "openaimmlu_sociology": { "original": 201, "effective": 201 }, "openaimmlu_high_school_macroeconomics": { "original": 390, "effective": 390 }, "openaimmlu_moral_scenarios": { "original": 895, "effective": 895 }, "openaimmlu_public_relations": { "original": 110, "effective": 110 }, "openaimmlu_us_foreign_policy": { "original": 100, "effective": 100 }, "openaimmlu_moral_disputes": { "original": 346, "effective": 346 }, "openaimmlu_human_sexuality": { "original": 131, "effective": 131 }, "openaimmlu_marketing": { "original": 234, "effective": 234 }, "openaimmlu_jurisprudence": { "original": 108, "effective": 108 }, "openaimmlu_logical_fallacies": { "original": 163, "effective": 163 }, "openaimmlu_philosophy": { "original": 311, "effective": 311 }, "openaimmlu_high_school_world_history": { "original": 237, "effective": 237 }, "openaimmlu_high_school_european_history": { "original": 165, "effective": 165 }, "openaimmlu_prehistory": { "original": 324, "effective": 324 }, "openaimmlu_high_school_us_history": { "original": 204, "effective": 204 }, "openaimmlu_international_law": { "original": 121, "effective": 121 }, "openaimmlu_world_religions": { "original": 171, "effective": 171 } }, "config": { "model": "vllm", "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8", "batch_size": 1, "batch_sizes": [], "device": null, "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": "8e1bd48d", "date": 1735663577.7452598, "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect", "transformers_version": "4.47.1", "upper_git_hash": null, "tokenizer_pad_token": [ "", "0" ], "tokenizer_eos_token": [ "", "2" ], "tokenizer_bos_token": [ "", "1" ], "eot_token_id": 2, "max_length": 4096, "task_hashes": { "openaimmlu_college_physics": "61aa73bd44d8ef4ba6cb739692c6eb8cabf49e8896a7f725463819ef0dbd0132", "openaimmlu_astronomy": "4c75961650ca77c7fb21671a45e42b30b2d6737dd89a9dd0f98b5a102a6fd21c", "openaimmlu_computer_security": "b3b48aa3be2791a75a9678e21c3f7000c0994934e0892e21be48b61eee9022b1", "openaimmlu_elementary_mathematics": "7ea44fa3e77564b6d8928cb20f739393b20c6df004e164290d5d90ef3d0a8b79", "openaimmlu_high_school_chemistry": "6d6a118469563d3ce711f5e5ca944e10ed6ea4e52c813217124fc043b7423db6", "openaimmlu_college_mathematics": "005761dd03c5fd7ac82e8a629717d9fa02e750f7f4913433240ae0886e421bc6", "openaimmlu_college_chemistry": "0539b9d96465df48c1370ac576a07d6d92f0829fe05cc79bd260ff763a74263d", "openaimmlu_college_biology": "ac595a195f3fe505c334d8ed12697594cafcbaca1d3247eb8d70a3562f41443e", "openaimmlu_conceptual_physics": "7e7cb338548eaf777c9eb7cca310fdf726660871c640842032735cc891816586", "openaimmlu_high_school_statistics": "94f1ab74a8bdbd75041a62e1855c3d15b6ade91a7cc96d274bf57c420c5e7a91", "openaimmlu_electrical_engineering": "599ce9a4e0938a3911259b3556952c70a5d9ac08df41625179cc73cb45a9797a", "openaimmlu_high_school_computer_science": "f89803071a28b442ab2f85f9dc6f5c3acc87118b662626c1e164aae4304f45ea", "openaimmlu_high_school_mathematics": "571309ff8e58fb1d5741d2d95ef005ff09d7e1fc61e75a59fd9bf77d1e4ec25e", "openaimmlu_abstract_algebra": "18718e53c9eb375b294dc89fddd44e9fec66166611545df741957cb9c3056597", "openaimmlu_high_school_physics": "b6a0c08c931f22af3809aba7b65315bf82834cf089961e2cada1bc6dab063306", "openaimmlu_college_computer_science": "614eb77451f839f693631aef6269e65c82e88ad3aa7105f665f4e6187723f986", "openaimmlu_econometrics": "5b060aa4148ab3c9e801d0591d391b90a21259c436d082120f19a16ce63f7c15", "openaimmlu_high_school_biology": "fbd661d888bdfd56e1256684914f1b2d2e90f128b26503e0e0d10af6af678e01", "openaimmlu_nutrition": "c44694a990c0a1187712f3e7d83ee10b5682fde624260c4b78bbd33641647f01", "openaimmlu_miscellaneous": "a42b3d1263bfa01552f44579362d25e558662731a595c2dec558d8c9ca4d727b", "openaimmlu_anatomy": "a2158278024b1e9f8867e30434721221f91cbcebaa34c3ca065cc039f6d9ce56", "openaimmlu_virology": "3fd44c94e0170284d5232b194c5604d338d0ace9cd0ff686d2349b0e7c2e19d6", "openaimmlu_professional_medicine": "c45c22a09efc77881a194f39b9622414eea01fdd59a4ce6fec12ca0bd542f73b", "openaimmlu_human_aging": "4216c0274bb171bdb7c8cc2640cd812401e292271f4ee2b95b73b73a48b061c1", "openaimmlu_clinical_knowledge": "3de23a26358560ceb58b2bd43bf1ca0873f1bde03f92f16048a7fe73ef086f3d", "openaimmlu_professional_accounting": "b08f816170cc1f742b5d62eca448427a3d57369d56d0db3349d79f0d9da3aec3", "openaimmlu_high_school_geography": "5f781f776d42c8b641139ad51da36b50da36c450bbec8a01fa33ae25a684133c", "openaimmlu_professional_psychology": "ee9b7102b9f931cb3c7fab155f7b1828160df589ae8b16844039de7ae3c8d064", "openaimmlu_high_school_psychology": "f62b828c33fd2ad378bb7aaf081c290210b378f9309aeef1fff01ddc83dd34c6", "openaimmlu_machine_learning": "4f9232ed92776579ea24eb5fe1ef1275bb2bf8a290f5f004f749cb64b6feee40", "openaimmlu_medical_genetics": "aece3b6adba4255559594b80438a0e80181e24366ff39b1c91456df945b01f4f", "openaimmlu_professional_law": "a86f6179dbad6d30286d1b71dafd2799ae6fa219cd7b3a079bb067483bc64b2a", "openaimmlu_college_medicine": "36232a0c51f93f761adf0529e7004f2f21479517d238b0c9bc538a1138e7482f", "openaimmlu_formal_logic": "5d7a76ba9f40981d143f6fe780a8269bd525cc54aea3fac3dde297b4d4491413", "openaimmlu_global_facts": "14d0542f6b985287da88bf4956de680e20754b9d74a009ca463beb66ae081e92", "openaimmlu_high_school_microeconomics": "51a598169dbd7a3de2e64558fd952a17cd39b49f0c6cb4de95fdb0e5520245c7", "openaimmlu_high_school_government_and_politics": "7c79983103e230916bf4d730743e92feb7f17e308a2897de764d24f49de93f37", "openaimmlu_management": "b85920acbd491f4bafeda9de9dc3af4408e63e0d53bfbeade834107dff6c3e3b", "openaimmlu_security_studies": "de9eccb24ee7d56897728b9bd30c0159f42a6cac86f3d2090504439efcbb2348", "openaimmlu_business_ethics": "5536730d841c70c256991081bd4d8a9c28aaebcab9ac3cdd36ceb1aad896cfae", "openaimmlu_sociology": "1fa4ebb60178be200e3b8167e10fec0843964725a6be034e6893ae42d5dd1a3a", "openaimmlu_high_school_macroeconomics": "938c0435e322f454a5b1f26f1b53870141a9e311bbc95512cf307ecd007e66db", "openaimmlu_moral_scenarios": "c0158287c824e917d6d76a9d4a4e68a53af20dee7bb7c1d372a759546edef562", "openaimmlu_public_relations": "545ea7d7b3dbbe04d2c367cd70142c35e1ce585a3c3e4b9d9fa2290d1d25272f", "openaimmlu_us_foreign_policy": "cddc366ef735093ff1ab6d3660a19d52ae146b9ca18668d8a878be81466cb626", "openaimmlu_moral_disputes": "2b9fd83448202cb343ad8473f9d34194776e73f9fac0fad093610033039e0152", "openaimmlu_human_sexuality": "cd4281e8629dd63b57e11fff680c2813bfc156d0807d9a3424670422bb8a8f02", "openaimmlu_marketing": "85aaada41a32346c0dce6f252b7e5e50a1bce1641cab3ecf6e1590deb8927db4", "openaimmlu_jurisprudence": "ab9de498411479a47a892895a70b20948854fe8c8177f9851da339a984c534f0", "openaimmlu_logical_fallacies": "1c809bb030ca1d7256a741cd2f7b3719053d6387df5f89762fca7aa430374461", "openaimmlu_philosophy": "e6367d4cab84d33e5ba62f20bc52f72d630c4324c0d34628b546cf72a83eb94f", "openaimmlu_high_school_world_history": "f7e18a11fcc4e11b8c758d3227d7e7fc59157c9bd465ade0c8e4707cb3c76c2a", "openaimmlu_high_school_european_history": "f6ee95e6dc273ad3d18c110a69772161a9eb250ef81c3202a46228689c5e2071", "openaimmlu_prehistory": "88d6574515b52d900aab10f1f22d026fa33c8d910a6528acf3dac384d1e82b0a", "openaimmlu_high_school_us_history": "541952c75bfb8c256d813fdfc4d7707ad25448980fd70d39142ec03a15af9d0d", "openaimmlu_international_law": "c1acfc8203c4d2f4d5d9245685804c2b7406601dfd8106cc4fac985915559f52", "openaimmlu_world_religions": "9b68b777a6bc2b05efee57f75e87792c6b14f39464621c16e4c24b024aeb2630" }, "model_source": "vllm", "model_name": "/tmp/7b-alpha-v1.27.2.25", "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25", "system_instruction": null, "system_instruction_sha": null, "fewshot_as_multiturn": false, "chat_template": null, "chat_template_sha": null, "start_time": 4238.00553465, "end_time": 4722.113520369, "total_evaluation_time_seconds": "484.10798571899977" }