1018 lines
70 KiB
Plaintext
1018 lines
70 KiB
Plaintext
[MMLU] start Sun Aug 9 22:52:08 UTC 2026
|
||
[*] MicroLLM2 MMLU — model: /home/zeus/microllm2/microllm2-checkpoints/final_merged shots=5 limit=20
|
||
[*] GPT2-XL 1.5B 1024ctx vocab=50259 (ChatML) — MMLU via direct eval (no harness needed)
|
||
[*] lm-eval not installed — using lightweight direct MMLU eval (same logic, no harness)
|
||
[*] Install harness for official numbers: pip install lm-eval==0.4.4
|
||
[*] Loading tokenizer + model /home/zeus/microllm2/microllm2-checkpoints/final_merged ...
|
||
[+] Loaded on cuda:0 dtype=torch.bfloat16 — starting MMLU
|
||
|
||
MMLU subjects: 0%| | 0/57 [00:00<?, ?it/s]
|
||
============================================================
|
||
[>] abstract_algebra (shots=5)
|
||
|
||
|
||
abstract_algebra: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
abstract_algebra: 5%|▌ | 1/20 [00:01<00:28, 1.50s/it][A
|
||
|
||
abstract_algebra: 15%|█▌ | 3/20 [00:01<00:07, 2.32it/s][A
|
||
|
||
abstract_algebra: 25%|██▌ | 5/20 [00:01<00:03, 4.18it/s][A
|
||
|
||
abstract_algebra: 40%|████ | 8/20 [00:01<00:01, 7.37it/s][A
|
||
|
||
abstract_algebra: 50%|█████ | 10/20 [00:01<00:01, 9.23it/s][A
|
||
|
||
abstract_algebra: 70%|███████ | 14/20 [00:02<00:00, 13.77it/s][A
|
||
|
||
abstract_algebra: 85%|████████▌ | 17/20 [00:02<00:00, 15.93it/s][A
|
||
|
||
abstract_algebra: 100%|██████████| 20/20 [00:02<00:00, 16.59it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 2%|▏ | 1/57 [00:03<03:40, 3.94s/it][=] abstract_algebra: 6/20 = 30.0% (running avg 30.0%)
|
||
|
||
============================================================
|
||
[>] anatomy (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/20.1k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 20.1k/20.1k [00:00<00:00, 60.9kB/s][A
|
||
Downloading data: 100%|██████████| 20.1k/20.1k [00:00<00:00, 60.8kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/5.28k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 5.28k/5.28k [00:00<00:00, 30.7kB/s][A
|
||
Downloading data: 100%|██████████| 5.28k/5.28k [00:00<00:00, 30.6kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/3.50k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 3.50k/3.50k [00:00<00:00, 18.8kB/s][A
|
||
Downloading data: 100%|██████████| 3.50k/3.50k [00:00<00:00, 18.7kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/135 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 135/135 [00:00<00:00, 20566.29 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/14 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 14/14 [00:00<00:00, 3828.67 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1496.15 examples/s]
|
||
|
||
|
||
anatomy: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
anatomy: 10%|█ | 2/20 [00:00<00:01, 14.47it/s][A
|
||
|
||
anatomy: 30%|███ | 6/20 [00:00<00:00, 26.90it/s][A
|
||
|
||
anatomy: 45%|████▌ | 9/20 [00:00<00:00, 25.51it/s][A
|
||
|
||
anatomy: 60%|██████ | 12/20 [00:00<00:00, 25.08it/s][A
|
||
|
||
anatomy: 75%|███████▌ | 15/20 [00:00<00:00, 24.24it/s][A
|
||
|
||
anatomy: 90%|█████████ | 18/20 [00:00<00:00, 22.08it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 4%|▎ | 2/57 [00:07<03:32, 3.87s/it][=] anatomy: 5/20 = 25.0% (running avg 27.5%)
|
||
|
||
============================================================
|
||
[>] astronomy (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/28.3k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 28.3k/28.3k [00:00<00:00, 154kB/s][A
|
||
Downloading data: 100%|██████████| 28.3k/28.3k [00:00<00:00, 153kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.05k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.05k/6.05k [00:00<00:00, 33.6kB/s][A
|
||
Downloading data: 100%|██████████| 6.05k/6.05k [00:00<00:00, 33.4kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.94k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.94k/4.94k [00:00<00:00, 28.3kB/s][A
|
||
Downloading data: 100%|██████████| 4.94k/4.94k [00:00<00:00, 28.2kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/152 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 152/152 [00:00<00:00, 30542.02 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/16 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 16/16 [00:00<00:00, 4092.75 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1423.63 examples/s]
|
||
|
||
|
||
astronomy: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
astronomy: 10%|█ | 2/20 [00:00<00:01, 12.56it/s][A
|
||
|
||
astronomy: 20%|██ | 4/20 [00:00<00:01, 14.83it/s][A
|
||
|
||
astronomy: 30%|███ | 6/20 [00:00<00:00, 16.13it/s][A
|
||
|
||
astronomy: 40%|████ | 8/20 [00:00<00:00, 16.72it/s][A
|
||
|
||
astronomy: 50%|█████ | 10/20 [00:00<00:00, 16.72it/s][A
|
||
|
||
astronomy: 60%|██████ | 12/20 [00:00<00:00, 17.03it/s][A
|
||
|
||
astronomy: 70%|███████ | 14/20 [00:00<00:00, 17.30it/s][A
|
||
|
||
astronomy: 85%|████████▌ | 17/20 [00:00<00:00, 19.30it/s][A
|
||
|
||
astronomy: 95%|█████████▌| 19/20 [00:01<00:00, 18.77it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 5%|▌ | 3/57 [00:11<03:27, 3.84s/it][=] astronomy: 7/20 = 35.0% (running avg 30.0%)
|
||
|
||
============================================================
|
||
[>] business_ethics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/21.6k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 21.6k/21.6k [00:00<00:00, 132kB/s][A
|
||
Downloading data: 100%|██████████| 21.6k/21.6k [00:00<00:00, 131kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/5.09k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 5.09k/5.09k [00:00<00:00, 34.0kB/s][A
|
||
Downloading data: 100%|██████████| 5.09k/5.09k [00:00<00:00, 33.8kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.96k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.96k/4.96k [00:00<00:00, 28.0kB/s][A
|
||
Downloading data: 100%|██████████| 4.96k/4.96k [00:00<00:00, 27.9kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 23243.58 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3018.27 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1394.75 examples/s]
|
||
|
||
|
||
business_ethics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
business_ethics: 10%|█ | 2/20 [00:00<00:01, 13.54it/s][A
|
||
|
||
business_ethics: 20%|██ | 4/20 [00:00<00:01, 15.82it/s][A
|
||
|
||
business_ethics: 40%|████ | 8/20 [00:00<00:00, 24.58it/s][A
|
||
|
||
business_ethics: 55%|█████▌ | 11/20 [00:00<00:00, 24.05it/s][A
|
||
|
||
business_ethics: 70%|███████ | 14/20 [00:00<00:00, 23.77it/s][A
|
||
|
||
business_ethics: 85%|████████▌ | 17/20 [00:00<00:00, 23.72it/s][A
|
||
|
||
business_ethics: 100%|██████████| 20/20 [00:00<00:00, 23.56it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 7%|▋ | 4/57 [00:15<03:28, 3.94s/it][=] business_ethics: 6/20 = 30.0% (running avg 30.0%)
|
||
|
||
============================================================
|
||
[>] clinical_knowledge (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/40.5k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 40.5k/40.5k [00:00<00:00, 199kB/s][A
|
||
Downloading data: 100%|██████████| 40.5k/40.5k [00:00<00:00, 199kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/7.48k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 7.48k/7.48k [00:00<00:00, 41.1kB/s][A
|
||
Downloading data: 100%|██████████| 7.48k/7.48k [00:00<00:00, 40.9kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/3.67k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 3.67k/3.67k [00:00<00:00, 20.4kB/s][A
|
||
Downloading data: 100%|██████████| 3.67k/3.67k [00:00<00:00, 20.4kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/265 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 265/265 [00:00<00:00, 52433.75 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/29 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 29/29 [00:00<00:00, 7840.33 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1485.34 examples/s]
|
||
|
||
|
||
clinical_knowledge: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
clinical_knowledge: 10%|█ | 2/20 [00:00<00:00, 18.25it/s][A
|
||
|
||
clinical_knowledge: 20%|██ | 4/20 [00:00<00:00, 18.30it/s][A
|
||
|
||
clinical_knowledge: 30%|███ | 6/20 [00:00<00:00, 18.27it/s][A
|
||
|
||
clinical_knowledge: 40%|████ | 8/20 [00:00<00:00, 17.59it/s][A
|
||
|
||
clinical_knowledge: 50%|█████ | 10/20 [00:00<00:00, 17.81it/s][A
|
||
|
||
clinical_knowledge: 60%|██████ | 12/20 [00:00<00:00, 17.79it/s][A
|
||
|
||
clinical_knowledge: 75%|███████▌ | 15/20 [00:00<00:00, 19.91it/s][A
|
||
|
||
clinical_knowledge: 90%|█████████ | 18/20 [00:00<00:00, 21.28it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 9%|▉ | 5/57 [00:19<03:24, 3.92s/it][=] clinical_knowledge: 9/20 = 45.0% (running avg 33.0%)
|
||
|
||
============================================================
|
||
[>] college_biology (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/31.8k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 31.8k/31.8k [00:00<00:00, 167kB/s][A
|
||
Downloading data: 100%|██████████| 31.8k/31.8k [00:00<00:00, 166kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.90k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.90k/6.90k [00:00<00:00, 51.0kB/s][A
|
||
Downloading data: 100%|██████████| 6.90k/6.90k [00:00<00:00, 50.7kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.27k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.27k/4.27k [00:00<00:00, 23.1kB/s][A
|
||
Downloading data: 100%|██████████| 4.27k/4.27k [00:00<00:00, 23.0kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/144 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 144/144 [00:00<00:00, 29006.81 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/16 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 16/16 [00:00<00:00, 4205.34 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1440.85 examples/s]
|
||
|
||
|
||
college_biology: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
college_biology: 10%|█ | 2/20 [00:00<00:01, 13.57it/s][A
|
||
|
||
college_biology: 20%|██ | 4/20 [00:00<00:00, 16.07it/s][A
|
||
|
||
college_biology: 35%|███▌ | 7/20 [00:00<00:00, 19.65it/s][A
|
||
|
||
college_biology: 45%|████▌ | 9/20 [00:00<00:00, 19.01it/s][A
|
||
|
||
college_biology: 65%|██████▌ | 13/20 [00:00<00:00, 22.13it/s][A
|
||
|
||
college_biology: 80%|████████ | 16/20 [00:00<00:00, 22.36it/s][A
|
||
|
||
college_biology: 95%|█████████▌| 19/20 [00:00<00:00, 22.63it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 11%|█ | 6/57 [00:23<03:14, 3.81s/it][=] college_biology: 9/20 = 45.0% (running avg 35.0%)
|
||
|
||
============================================================
|
||
[>] college_chemistry (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/17.9k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 17.9k/17.9k [00:00<00:00, 97.0kB/s][A
|
||
Downloading data: 100%|██████████| 17.9k/17.9k [00:00<00:00, 96.4kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.87k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.87k/4.87k [00:00<00:00, 27.0kB/s][A
|
||
Downloading data: 100%|██████████| 4.87k/4.87k [00:00<00:00, 26.9kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.04k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.04k/4.04k [00:00<00:00, 10.3kB/s][A
|
||
Downloading data: 100%|██████████| 4.04k/4.04k [00:00<00:00, 10.2kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 22971.16 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/8 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 8/8 [00:00<00:00, 2089.45 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1161.02 examples/s]
|
||
|
||
|
||
college_chemistry: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
college_chemistry: 10%|█ | 2/20 [00:00<00:01, 12.89it/s][A
|
||
|
||
college_chemistry: 25%|██▌ | 5/20 [00:00<00:00, 18.24it/s][A
|
||
|
||
college_chemistry: 40%|████ | 8/20 [00:00<00:00, 20.45it/s][A
|
||
|
||
college_chemistry: 55%|█████▌ | 11/20 [00:00<00:00, 19.28it/s][A
|
||
|
||
college_chemistry: 65%|██████▌ | 13/20 [00:00<00:00, 18.81it/s][A
|
||
|
||
college_chemistry: 85%|████████▌ | 17/20 [00:00<00:00, 21.88it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 12%|█▏ | 7/57 [00:27<03:14, 3.88s/it][=] college_chemistry: 3/20 = 15.0% (running avg 32.1%)
|
||
|
||
============================================================
|
||
[>] college_computer_science (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/28.1k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 28.1k/28.1k [00:00<00:00, 164kB/s][A
|
||
Downloading data: 100%|██████████| 28.1k/28.1k [00:00<00:00, 163kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.25k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.25k/6.25k [00:00<00:00, 40.5kB/s][A
|
||
Downloading data: 100%|██████████| 6.25k/6.25k [00:00<00:00, 40.2kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.81k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.81k/6.81k [00:00<00:00, 46.9kB/s][A
|
||
Downloading data: 100%|██████████| 6.81k/6.81k [00:00<00:00, 46.6kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 22972.42 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3038.75 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1146.55 examples/s]
|
||
|
||
|
||
college_computer_science: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
college_computer_science: 10%|█ | 2/20 [00:00<00:01, 12.20it/s][A
|
||
|
||
college_computer_science: 20%|██ | 4/20 [00:00<00:01, 13.86it/s][A
|
||
|
||
college_computer_science: 30%|███ | 6/20 [00:00<00:00, 15.04it/s][A
|
||
|
||
college_computer_science: 40%|████ | 8/20 [00:00<00:00, 15.92it/s][A
|
||
|
||
college_computer_science: 50%|█████ | 10/20 [00:00<00:00, 16.45it/s][A
|
||
|
||
college_computer_science: 60%|██████ | 12/20 [00:00<00:00, 16.91it/s][A
|
||
|
||
college_computer_science: 70%|███████ | 14/20 [00:00<00:00, 17.07it/s][A
|
||
|
||
college_computer_science: 80%|████████ | 16/20 [00:00<00:00, 16.99it/s][A
|
||
|
||
college_computer_science: 90%|█████████ | 18/20 [00:01<00:00, 17.08it/s][A
|
||
|
||
college_computer_science: 100%|██████████| 20/20 [00:01<00:00, 17.00it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 14%|█▍ | 8/57 [00:31<03:10, 3.88s/it][=] college_computer_science: 9/20 = 45.0% (running avg 33.8%)
|
||
|
||
============================================================
|
||
[>] college_mathematics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/16.6k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 16.6k/16.6k [00:00<00:00, 85.6kB/s][A
|
||
Downloading data: 100%|██████████| 16.6k/16.6k [00:00<00:00, 85.2kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/5.00k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 5.00k/5.00k [00:00<00:00, 26.9kB/s][A
|
||
Downloading data: 100%|██████████| 5.00k/5.00k [00:00<00:00, 26.8kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/5.16k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 5.16k/5.16k [00:00<00:00, 36.3kB/s][A
|
||
Downloading data: 100%|██████████| 5.16k/5.16k [00:00<00:00, 36.1kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 22919.69 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3062.55 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1144.17 examples/s]
|
||
|
||
|
||
college_mathematics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
college_mathematics: 10%|█ | 2/20 [00:00<00:01, 13.57it/s][A
|
||
|
||
college_mathematics: 20%|██ | 4/20 [00:00<00:01, 15.77it/s][A
|
||
|
||
college_mathematics: 35%|███▌ | 7/20 [00:00<00:00, 19.18it/s][A
|
||
|
||
college_mathematics: 60%|██████ | 12/20 [00:00<00:00, 27.17it/s][A
|
||
|
||
college_mathematics: 80%|████████ | 16/20 [00:00<00:00, 27.40it/s][A
|
||
|
||
college_mathematics: 95%|█████████▌| 19/20 [00:00<00:00, 23.83it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 16%|█▌ | 9/57 [00:34<03:01, 3.79s/it][=] college_mathematics: 7/20 = 35.0% (running avg 33.9%)
|
||
|
||
============================================================
|
||
[>] college_medicine (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/42.5k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 42.5k/42.5k [00:00<00:00, 221kB/s][A
|
||
Downloading data: 100%|██████████| 42.5k/42.5k [00:00<00:00, 220kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/8.99k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 8.99k/8.99k [00:00<00:00, 54.2kB/s][A
|
||
Downloading data: 100%|██████████| 8.99k/8.99k [00:00<00:00, 53.9kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.84k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.84k/4.84k [00:00<00:00, 34.4kB/s][A
|
||
Downloading data: 100%|██████████| 4.84k/4.84k [00:00<00:00, 34.1kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/173 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 173/173 [00:00<00:00, 34671.95 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/22 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 22/22 [00:00<00:00, 4797.23 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1361.43 examples/s]
|
||
|
||
|
||
college_medicine: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
college_medicine: 10%|█ | 2/20 [00:00<00:00, 18.63it/s][A
|
||
|
||
college_medicine: 20%|██ | 4/20 [00:00<00:00, 17.33it/s][A
|
||
|
||
college_medicine: 30%|███ | 6/20 [00:00<00:00, 17.41it/s][A
|
||
|
||
college_medicine: 45%|████▌ | 9/20 [00:00<00:00, 19.76it/s][A
|
||
|
||
college_medicine: 55%|█████▌ | 11/20 [00:00<00:00, 18.87it/s][A
|
||
|
||
college_medicine: 65%|██████▌ | 13/20 [00:00<00:00, 18.40it/s][A
|
||
|
||
college_medicine: 75%|███████▌ | 15/20 [00:00<00:00, 18.19it/s][A
|
||
|
||
college_medicine: 100%|██████████| 20/20 [00:00<00:00, 25.63it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 18%|█▊ | 10/57 [00:38<02:55, 3.74s/it][=] college_medicine: 6/20 = 30.0% (running avg 33.5%)
|
||
|
||
============================================================
|
||
[>] college_physics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/18.6k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 18.6k/18.6k [00:00<00:00, 103kB/s][A
|
||
Downloading data: 100%|██████████| 18.6k/18.6k [00:00<00:00, 102kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.39k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.39k/6.39k [00:00<00:00, 39.7kB/s][A
|
||
Downloading data: 100%|██████████| 6.39k/6.39k [00:00<00:00, 39.5kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.51k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.51k/4.51k [00:00<00:00, 24.1kB/s][A
|
||
Downloading data: 100%|██████████| 4.51k/4.51k [00:00<00:00, 23.9kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/102 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 102/102 [00:00<00:00, 17934.14 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 2286.18 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1061.58 examples/s]
|
||
|
||
|
||
college_physics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
college_physics: 10%|█ | 2/20 [00:00<00:01, 13.64it/s][A
|
||
|
||
college_physics: 20%|██ | 4/20 [00:00<00:01, 14.37it/s][A
|
||
|
||
college_physics: 35%|███▌ | 7/20 [00:00<00:00, 19.91it/s][A
|
||
|
||
college_physics: 50%|█████ | 10/20 [00:00<00:00, 21.23it/s][A
|
||
|
||
college_physics: 70%|███████ | 14/20 [00:00<00:00, 23.58it/s][A
|
||
|
||
college_physics: 90%|█████████ | 18/20 [00:00<00:00, 27.68it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 19%|█▉ | 11/57 [00:42<02:55, 3.82s/it][=] college_physics: 3/20 = 15.0% (running avg 31.8%)
|
||
|
||
============================================================
|
||
[>] computer_security (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/19.1k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 19.1k/19.1k [00:00<00:00, 106kB/s][A
|
||
Downloading data: 100%|██████████| 19.1k/19.1k [00:00<00:00, 106kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.67k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.67k/6.67k [00:00<00:00, 46.8kB/s][A
|
||
Downloading data: 100%|██████████| 6.67k/6.67k [00:00<00:00, 46.6kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.33k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.33k/4.33k [00:00<00:00, 23.8kB/s][A
|
||
Downloading data: 100%|██████████| 4.33k/4.33k [00:00<00:00, 23.7kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 22049.75 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 2974.88 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1465.82 examples/s]
|
||
|
||
|
||
computer_security: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
computer_security: 10%|█ | 2/20 [00:00<00:00, 18.58it/s][A
|
||
|
||
computer_security: 20%|██ | 4/20 [00:00<00:00, 18.42it/s][A
|
||
|
||
computer_security: 35%|███▌ | 7/20 [00:00<00:00, 21.21it/s][A
|
||
|
||
computer_security: 55%|█████▌ | 11/20 [00:00<00:00, 24.40it/s][A
|
||
|
||
computer_security: 70%|███████ | 14/20 [00:00<00:00, 24.22it/s][A
|
||
|
||
computer_security: 90%|█████████ | 18/20 [00:00<00:00, 28.44it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 21%|██ | 12/57 [00:45<02:48, 3.75s/it][=] computer_security: 6/20 = 30.0% (running avg 31.7%)
|
||
|
||
============================================================
|
||
[>] conceptual_physics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/25.0k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 25.0k/25.0k [00:00<00:00, 153kB/s][A
|
||
Downloading data: 100%|██████████| 25.0k/25.0k [00:00<00:00, 152kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/5.98k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 5.98k/5.98k [00:00<00:00, 32.3kB/s][A
|
||
Downloading data: 100%|██████████| 5.98k/5.98k [00:00<00:00, 32.1kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/3.96k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 3.96k/3.96k [00:00<00:00, 22.4kB/s][A
|
||
Downloading data: 100%|██████████| 3.96k/3.96k [00:00<00:00, 22.3kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/235 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 235/235 [00:00<00:00, 51400.78 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/26 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 26/26 [00:00<00:00, 6864.22 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1387.28 examples/s]
|
||
|
||
|
||
conceptual_physics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
conceptual_physics: 10%|█ | 2/20 [00:00<00:01, 13.26it/s][A
|
||
|
||
conceptual_physics: 25%|██▌ | 5/20 [00:00<00:00, 18.87it/s][A
|
||
|
||
conceptual_physics: 35%|███▌ | 7/20 [00:00<00:00, 18.34it/s][A
|
||
|
||
conceptual_physics: 50%|█████ | 10/20 [00:00<00:00, 19.91it/s][A
|
||
|
||
conceptual_physics: 70%|███████ | 14/20 [00:00<00:00, 25.59it/s][A
|
||
|
||
conceptual_physics: 85%|████████▌ | 17/20 [00:00<00:00, 24.89it/s][A
|
||
|
||
conceptual_physics: 100%|██████████| 20/20 [00:00<00:00, 24.58it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 23%|██▎ | 13/57 [00:49<02:40, 3.66s/it][=] conceptual_physics: 1/20 = 5.0% (running avg 29.6%)
|
||
|
||
============================================================
|
||
[>] econometrics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/24.5k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 24.5k/24.5k [00:00<00:00, 141kB/s][A
|
||
Downloading data: 100%|██████████| 24.5k/24.5k [00:00<00:00, 140kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/7.02k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 7.02k/7.02k [00:00<00:00, 26.7kB/s][A
|
||
Downloading data: 100%|██████████| 7.02k/7.02k [00:00<00:00, 26.7kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.54k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.54k/4.54k [00:00<00:00, 29.1kB/s][A
|
||
Downloading data: 100%|██████████| 4.54k/4.54k [00:00<00:00, 28.9kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/114 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 114/114 [00:00<00:00, 22894.45 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/12 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 12/12 [00:00<00:00, 3162.13 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1402.87 examples/s]
|
||
|
||
|
||
econometrics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
econometrics: 20%|██ | 4/20 [00:00<00:00, 29.13it/s][A
|
||
|
||
econometrics: 35%|███▌ | 7/20 [00:00<00:00, 22.20it/s][A
|
||
|
||
econometrics: 55%|█████▌ | 11/20 [00:00<00:00, 24.18it/s][A
|
||
|
||
econometrics: 80%|████████ | 16/20 [00:00<00:00, 29.48it/s][A
|
||
|
||
econometrics: 100%|██████████| 20/20 [00:00<00:00, 28.37it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 25%|██▍ | 14/57 [00:53<02:39, 3.70s/it][=] econometrics: 6/20 = 30.0% (running avg 29.6%)
|
||
|
||
============================================================
|
||
[>] electrical_engineering (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/17.6k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 17.6k/17.6k [00:00<00:00, 108kB/s][A
|
||
Downloading data: 100%|██████████| 17.6k/17.6k [00:00<00:00, 107kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/5.08k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 5.08k/5.08k [00:00<00:00, 35.0kB/s][A
|
||
Downloading data: 100%|██████████| 5.08k/5.08k [00:00<00:00, 34.7kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.08k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.08k/4.08k [00:00<00:00, 22.8kB/s][A
|
||
Downloading data: 100%|██████████| 4.08k/4.08k [00:00<00:00, 22.7kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/145 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 145/145 [00:00<00:00, 33690.12 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/16 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 16/16 [00:00<00:00, 4531.63 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1402.68 examples/s]
|
||
|
||
|
||
electrical_engineering: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
electrical_engineering: 10%|█ | 2/20 [00:00<00:00, 19.15it/s][A
|
||
|
||
electrical_engineering: 30%|███ | 6/20 [00:00<00:00, 24.71it/s][A
|
||
|
||
electrical_engineering: 45%|████▌ | 9/20 [00:00<00:00, 21.26it/s][A
|
||
|
||
electrical_engineering: 60%|██████ | 12/20 [00:00<00:00, 22.72it/s][A
|
||
|
||
electrical_engineering: 90%|█████████ | 18/20 [00:00<00:00, 33.51it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 26%|██▋ | 15/57 [00:56<02:30, 3.59s/it][=] electrical_engineering: 4/20 = 20.0% (running avg 29.0%)
|
||
|
||
============================================================
|
||
[>] elementary_mathematics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/41.1k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 41.1k/41.1k [00:00<00:00, 223kB/s][A
|
||
Downloading data: 100%|██████████| 41.1k/41.1k [00:00<00:00, 222kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/9.38k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 9.38k/9.38k [00:00<00:00, 52.5kB/s][A
|
||
Downloading data: 100%|██████████| 9.38k/9.38k [00:00<00:00, 52.3kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.55k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.55k/4.55k [00:00<00:00, 21.6kB/s][A
|
||
Downloading data: 100%|██████████| 4.55k/4.55k [00:00<00:00, 21.6kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/378 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 378/378 [00:00<00:00, 78826.97 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/41 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 41/41 [00:00<00:00, 11570.88 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1528.09 examples/s]
|
||
|
||
|
||
elementary_mathematics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
elementary_mathematics: 10%|█ | 2/20 [00:00<00:01, 13.75it/s][A
|
||
|
||
elementary_mathematics: 20%|██ | 4/20 [00:00<00:00, 16.16it/s][A
|
||
|
||
elementary_mathematics: 30%|███ | 6/20 [00:00<00:00, 16.90it/s][A
|
||
|
||
elementary_mathematics: 50%|█████ | 10/20 [00:00<00:00, 24.46it/s][A
|
||
|
||
elementary_mathematics: 70%|███████ | 14/20 [00:00<00:00, 28.79it/s][A
|
||
|
||
elementary_mathematics: 85%|████████▌ | 17/20 [00:00<00:00, 26.72it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 28%|██▊ | 16/57 [00:59<02:26, 3.56s/it][=] elementary_mathematics: 6/20 = 30.0% (running avg 29.1%)
|
||
|
||
============================================================
|
||
[>] formal_logic (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/21.5k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 21.5k/21.5k [00:00<00:00, 111kB/s][A
|
||
Downloading data: 100%|██████████| 21.5k/21.5k [00:00<00:00, 110kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.56k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.56k/6.56k [00:00<00:00, 39.0kB/s][A
|
||
Downloading data: 100%|██████████| 6.56k/6.56k [00:00<00:00, 38.8kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.81k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.81k/4.81k [00:00<00:00, 22.4kB/s][A
|
||
Downloading data: 100%|██████████| 4.81k/4.81k [00:00<00:00, 22.3kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/126 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 126/126 [00:00<00:00, 28698.47 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/14 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 14/14 [00:00<00:00, 3785.72 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1486.81 examples/s]
|
||
|
||
|
||
formal_logic: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
formal_logic: 10%|█ | 2/20 [00:00<00:01, 13.48it/s][A
|
||
|
||
formal_logic: 20%|██ | 4/20 [00:00<00:01, 15.63it/s][A
|
||
|
||
formal_logic: 30%|███ | 6/20 [00:00<00:00, 16.30it/s][A
|
||
|
||
formal_logic: 45%|████▌ | 9/20 [00:00<00:00, 19.01it/s][A
|
||
|
||
formal_logic: 65%|██████▌ | 13/20 [00:00<00:00, 22.36it/s][A
|
||
|
||
formal_logic: 80%|████████ | 16/20 [00:00<00:00, 22.63it/s][A
|
||
|
||
formal_logic: 95%|█████████▌| 19/20 [00:00<00:00, 22.82it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 30%|██▉ | 17/57 [01:03<02:23, 3.58s/it][=] formal_logic: 2/20 = 10.0% (running avg 27.9%)
|
||
|
||
============================================================
|
||
[>] global_facts (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/11.5k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 11.5k/11.5k [00:00<00:00, 59.0kB/s][A
|
||
Downloading data: 100%|██████████| 11.5k/11.5k [00:00<00:00, 58.8kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.19k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.19k/4.19k [00:00<00:00, 22.6kB/s][A
|
||
Downloading data: 100%|██████████| 4.19k/4.19k [00:00<00:00, 22.5kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/3.58k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 3.58k/3.58k [00:00<00:00, 24.2kB/s][A
|
||
Downloading data: 100%|██████████| 3.58k/3.58k [00:00<00:00, 24.1kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 22527.01 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/10 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 10/10 [00:00<00:00, 2616.37 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1165.93 examples/s]
|
||
|
||
|
||
global_facts: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
global_facts: 10%|█ | 2/20 [00:00<00:00, 19.36it/s][A
|
||
|
||
global_facts: 30%|███ | 6/20 [00:00<00:00, 30.23it/s][A
|
||
|
||
global_facts: 50%|█████ | 10/20 [00:00<00:00, 33.55it/s][A
|
||
|
||
global_facts: 70%|███████ | 14/20 [00:00<00:00, 34.95it/s][A
|
||
|
||
global_facts: 100%|██████████| 20/20 [00:00<00:00, 43.53it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 32%|███▏ | 18/57 [01:06<02:15, 3.47s/it][=] global_facts: 7/20 = 35.0% (running avg 28.3%)
|
||
|
||
============================================================
|
||
[>] high_school_biology (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/62.7k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 62.7k/62.7k [00:00<00:00, 355kB/s][A
|
||
Downloading data: 100%|██████████| 62.7k/62.7k [00:00<00:00, 353kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/10.6k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 10.6k/10.6k [00:00<00:00, 58.4kB/s][A
|
||
Downloading data: 100%|██████████| 10.6k/10.6k [00:00<00:00, 58.2kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.94k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.94k/4.94k [00:00<00:00, 29.3kB/s][A
|
||
Downloading data: 100%|██████████| 4.94k/4.94k [00:00<00:00, 29.2kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/310 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 310/310 [00:00<00:00, 60104.20 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/32 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 32/32 [00:00<00:00, 7002.18 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1401.37 examples/s]
|
||
|
||
|
||
high_school_biology: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
high_school_biology: 15%|█▌ | 3/20 [00:00<00:00, 24.67it/s][A
|
||
|
||
high_school_biology: 35%|███▌ | 7/20 [00:00<00:00, 31.39it/s][A
|
||
|
||
high_school_biology: 55%|█████▌ | 11/20 [00:00<00:00, 25.25it/s][A
|
||
|
||
high_school_biology: 75%|███████▌ | 15/20 [00:00<00:00, 25.85it/s][A
|
||
|
||
high_school_biology: 95%|█████████▌| 19/20 [00:00<00:00, 29.14it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 33%|███▎ | 19/57 [01:10<02:10, 3.42s/it][=] high_school_biology: 9/20 = 45.0% (running avg 29.2%)
|
||
|
||
============================================================
|
||
[>] high_school_chemistry (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/33.3k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 33.3k/33.3k [00:00<00:00, 186kB/s][A
|
||
Downloading data: 100%|██████████| 33.3k/33.3k [00:00<00:00, 185kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/8.31k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 8.31k/8.31k [00:00<00:00, 59.6kB/s][A
|
||
Downloading data: 100%|██████████| 8.31k/8.31k [00:00<00:00, 59.2kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.16k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.16k/4.16k [00:00<00:00, 20.8kB/s][A
|
||
Downloading data: 100%|██████████| 4.16k/4.16k [00:00<00:00, 20.7kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/203 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 203/203 [00:00<00:00, 43367.99 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/22 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 22/22 [00:00<00:00, 5422.50 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1380.34 examples/s]
|
||
|
||
|
||
high_school_chemistry: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
high_school_chemistry: 10%|█ | 2/20 [00:00<00:00, 18.19it/s][A
|
||
|
||
high_school_chemistry: 40%|████ | 8/20 [00:00<00:00, 41.63it/s][A
|
||
|
||
high_school_chemistry: 65%|██████▌ | 13/20 [00:00<00:00, 41.11it/s][A
|
||
|
||
high_school_chemistry: 90%|█████████ | 18/20 [00:00<00:00, 41.15it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 35%|███▌ | 20/57 [01:13<02:05, 3.40s/it][=] high_school_chemistry: 7/20 = 35.0% (running avg 29.5%)
|
||
|
||
============================================================
|
||
[>] high_school_computer_science (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/27.3k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 27.3k/27.3k [00:00<00:00, 174kB/s][A
|
||
Downloading data: 100%|██████████| 27.3k/27.3k [00:00<00:00, 173kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/5.28k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 5.28k/5.28k [00:00<00:00, 36.1kB/s][A
|
||
Downloading data: 100%|██████████| 5.28k/5.28k [00:00<00:00, 36.0kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.54k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.54k/6.54k [00:00<00:00, 36.2kB/s][A
|
||
Downloading data: 100%|██████████| 6.54k/6.54k [00:00<00:00, 36.0kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 22050.91 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/9 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 9/9 [00:00<00:00, 2392.04 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1416.80 examples/s]
|
||
|
||
|
||
high_school_computer_science: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
high_school_computer_science: 10%|█ | 2/20 [00:00<00:01, 13.37it/s][A
|
||
|
||
high_school_computer_science: 30%|███ | 6/20 [00:00<00:00, 20.73it/s][A
|
||
|
||
high_school_computer_science: 50%|█████ | 10/20 [00:00<00:00, 26.18it/s][A
|
||
|
||
high_school_computer_science: 65%|██████▌ | 13/20 [00:00<00:00, 24.77it/s][A
|
||
|
||
high_school_computer_science: 80%|████████ | 16/20 [00:00<00:00, 23.89it/s][A
|
||
|
||
high_school_computer_science: 95%|█████████▌| 19/20 [00:00<00:00, 23.39it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 37%|███▋ | 21/57 [01:16<02:03, 3.43s/it][=] high_school_computer_science: 7/20 = 35.0% (running avg 29.8%)
|
||
|
||
============================================================
|
||
[>] high_school_european_history (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/142k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 142k/142k [00:00<00:00, 681kB/s][A
|
||
Downloading data: 100%|██████████| 142k/142k [00:00<00:00, 678kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/31.6k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 31.6k/31.6k [00:00<00:00, 202kB/s][A
|
||
Downloading data: 100%|██████████| 31.6k/31.6k [00:00<00:00, 201kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/22.2k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 22.2k/22.2k [00:00<00:00, 113kB/s][A
|
||
Downloading data: 100%|██████████| 22.2k/22.2k [00:00<00:00, 112kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/165 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 165/165 [00:00<00:00, 31161.25 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/18 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 18/18 [00:00<00:00, 4798.67 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1341.75 examples/s]
|
||
|
||
|
||
high_school_european_history: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
high_school_european_history: 20%|██ | 4/20 [00:00<00:00, 34.23it/s][A
|
||
|
||
high_school_european_history: 45%|████▌ | 9/20 [00:00<00:00, 41.23it/s][A
|
||
|
||
high_school_european_history: 70%|███████ | 14/20 [00:00<00:00, 43.45it/s][A
|
||
|
||
high_school_european_history: 95%|█████████▌| 19/20 [00:00<00:00, 44.75it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 39%|███▊ | 22/57 [01:20<01:57, 3.36s/it][=] high_school_european_history: 4/20 = 20.0% (running avg 29.3%)
|
||
|
||
============================================================
|
||
[>] high_school_geography (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/28.2k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 28.2k/28.2k [00:00<00:00, 146kB/s][A
|
||
Downloading data: 100%|██████████| 28.2k/28.2k [00:00<00:00, 145kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.16k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.16k/6.16k [00:00<00:00, 40.8kB/s][A
|
||
Downloading data: 100%|██████████| 6.16k/6.16k [00:00<00:00, 40.6kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/3.93k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 3.93k/3.93k [00:00<00:00, 28.5kB/s][A
|
||
Downloading data: 100%|██████████| 3.93k/3.93k [00:00<00:00, 28.4kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/198 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 198/198 [00:00<00:00, 40891.83 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/22 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 22/22 [00:00<00:00, 5636.82 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1341.75 examples/s]
|
||
|
||
|
||
high_school_geography: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
high_school_geography: 15%|█▌ | 3/20 [00:00<00:00, 24.44it/s][A
|
||
|
||
high_school_geography: 45%|████▌ | 9/20 [00:00<00:00, 35.42it/s][A
|
||
|
||
high_school_geography: 65%|██████▌ | 13/20 [00:00<00:00, 31.26it/s][A
|
||
|
||
high_school_geography: 85%|████████▌ | 17/20 [00:00<00:00, 33.58it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 40%|████ | 23/57 [01:23<01:53, 3.32s/it][=] high_school_geography: 5/20 = 25.0% (running avg 29.1%)
|
||
|
||
============================================================
|
||
[>] high_school_government_and_politics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/40.2k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 40.2k/40.2k [00:00<00:00, 277kB/s][A
|
||
Downloading data: 100%|██████████| 40.2k/40.2k [00:00<00:00, 276kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/8.27k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 8.27k/8.27k [00:00<00:00, 54.1kB/s][A
|
||
Downloading data: 100%|██████████| 8.27k/8.27k [00:00<00:00, 53.9kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.47k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.47k/4.47k [00:00<00:00, 26.2kB/s][A
|
||
Downloading data: 100%|██████████| 4.47k/4.47k [00:00<00:00, 26.1kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/193 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 193/193 [00:00<00:00, 40098.11 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/21 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 21/21 [00:00<00:00, 5715.79 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1422.28 examples/s]
|
||
|
||
|
||
high_school_government_and_politics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
high_school_government_and_politics: 10%|█ | 2/20 [00:00<00:00, 18.68it/s][A
|
||
|
||
high_school_government_and_politics: 30%|███ | 6/20 [00:00<00:00, 24.23it/s][A
|
||
|
||
high_school_government_and_politics: 60%|██████ | 12/20 [00:00<00:00, 37.92it/s][A
|
||
|
||
high_school_government_and_politics: 90%|█████████ | 18/20 [00:00<00:00, 39.98it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 42%|████▏ | 24/57 [01:26<01:46, 3.23s/it][=] high_school_government_and_politics: 4/20 = 20.0% (running avg 28.7%)
|
||
|
||
============================================================
|
||
[>] high_school_macroeconomics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/54.8k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 54.8k/54.8k [00:00<00:00, 364kB/s][A
|
||
Downloading data: 100%|██████████| 54.8k/54.8k [00:00<00:00, 362kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/9.89k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 9.89k/9.89k [00:00<00:00, 50.4kB/s][A
|
||
Downloading data: 100%|██████████| 9.89k/9.89k [00:00<00:00, 50.2kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.04k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.04k/4.04k [00:00<00:00, 29.8kB/s][A
|
||
Downloading data: 100%|██████████| 4.04k/4.04k [00:00<00:00, 29.7kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/390 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 390/390 [00:00<00:00, 70428.77 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/43 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 43/43 [00:00<00:00, 10985.20 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1389.95 examples/s]
|
||
|
||
|
||
high_school_macroeconomics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
high_school_macroeconomics: 25%|██▌ | 5/20 [00:00<00:00, 32.92it/s][A
|
||
|
||
high_school_macroeconomics: 55%|█████▌ | 11/20 [00:00<00:00, 39.10it/s][A
|
||
|
||
high_school_macroeconomics: 75%|███████▌ | 15/20 [00:00<00:00, 38.51it/s][A
|
||
|
||
high_school_macroeconomics: 95%|█████████▌| 19/20 [00:00<00:00, 38.22it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 44%|████▍ | 25/57 [01:29<01:43, 3.24s/it][=] high_school_macroeconomics: 0/20 = 0.0% (running avg 27.6%)
|
||
|
||
============================================================
|
||
[>] high_school_mathematics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/33.7k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 33.7k/33.7k [00:00<00:00, 219kB/s][A
|
||
Downloading data: 100%|██████████| 33.7k/33.7k [00:00<00:00, 218kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/6.99k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 6.99k/6.99k [00:00<00:00, 28.4kB/s][A
|
||
Downloading data: 100%|██████████| 6.99k/6.99k [00:00<00:00, 28.3kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/4.50k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 4.50k/4.50k [00:00<00:00, 21.2kB/s][A
|
||
Downloading data: 100%|██████████| 4.50k/4.50k [00:00<00:00, 21.2kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/270 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 270/270 [00:00<00:00, 59139.49 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/29 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 29/29 [00:00<00:00, 8262.11 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1471.58 examples/s]
|
||
|
||
|
||
high_school_mathematics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
high_school_mathematics: 10%|█ | 2/20 [00:00<00:01, 13.78it/s][A
|
||
|
||
high_school_mathematics: 30%|███ | 6/20 [00:00<00:00, 21.99it/s][A
|
||
|
||
high_school_mathematics: 65%|██████▌ | 13/20 [00:00<00:00, 37.38it/s][A
|
||
|
||
high_school_mathematics: 95%|█████████▌| 19/20 [00:00<00:00, 39.66it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 46%|████▌ | 26/57 [01:33<01:42, 3.29s/it][=] high_school_mathematics: 4/20 = 20.0% (running avg 27.3%)
|
||
|
||
============================================================
|
||
[>] high_school_microeconomics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/38.8k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 38.8k/38.8k [00:00<00:00, 267kB/s][A
|
||
Downloading data: 100%|██████████| 38.8k/38.8k [00:00<00:00, 266kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/7.22k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 7.22k/7.22k [00:00<00:00, 42.1kB/s][A
|
||
Downloading data: 100%|██████████| 7.22k/7.22k [00:00<00:00, 41.9kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/3.83k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 3.83k/3.83k [00:00<00:00, 25.1kB/s][A
|
||
Downloading data: 100%|██████████| 3.83k/3.83k [00:00<00:00, 25.0kB/s]
|
||
|
||
|
||
Generating test split: 0%| | 0/238 [00:00<?, ? examples/s][A
|
||
Generating test split: 100%|██████████| 238/238 [00:00<00:00, 49842.44 examples/s]
|
||
|
||
|
||
Generating validation split: 0%| | 0/26 [00:00<?, ? examples/s][A
|
||
Generating validation split: 100%|██████████| 26/26 [00:00<00:00, 7166.92 examples/s]
|
||
|
||
|
||
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s][A
|
||
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1446.41 examples/s]
|
||
|
||
|
||
high_school_microeconomics: 0%| | 0/20 [00:00<?, ?it/s][A
|
||
|
||
high_school_microeconomics: 15%|█▌ | 3/20 [00:00<00:00, 25.04it/s][A
|
||
|
||
high_school_microeconomics: 35%|███▌ | 7/20 [00:00<00:00, 32.17it/s][A
|
||
|
||
high_school_microeconomics: 55%|█████▌ | 11/20 [00:00<00:00, 34.15it/s][A
|
||
|
||
high_school_microeconomics: 75%|███████▌ | 15/20 [00:00<00:00, 35.22it/s][A
|
||
|
||
high_school_microeconomics: 95%|█████████▌| 19/20 [00:00<00:00, 31.88it/s][A
|
||
|
||
[A
|
||
MMLU subjects: 47%|████▋ | 27/57 [01:36<01:37, 3.25s/it][=] high_school_microeconomics: 7/20 = 35.0% (running avg 27.6%)
|
||
|
||
============================================================
|
||
[>] high_school_physics (shots=5)
|
||
|
||
|
||
Downloading data: 0%| | 0.00/33.0k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 33.0k/33.0k [00:00<00:00, 197kB/s][A
|
||
Downloading data: 100%|██████████| 33.0k/33.0k [00:00<00:00, 196kB/s]
|
||
|
||
|
||
Downloading data: 0%| | 0.00/7.96k [00:00<?, ?B/s][A
|
||
|
||
Downloading data: 100%|██████████| 7.96k/7.96k [00:00<00:00, 42.4kB/s][A
|
||
Downloading data: 100%|██████████| 7.96k/7.96k [00:00<00:00, 42.2kB/s]
|
||
|
||
MMLU subjects: 49%|████▉ | 28/57 [01:38<01:27, 3.02s/it]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 51%|█████ | 29/57 [01:38<01:00, 2.15s/it]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 54%|█████▍ | 31/57 [01:39<00:31, 1.20s/it]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 56%|█████▌ | 32/57 [01:39<00:23, 1.08it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 58%|█████▊ | 33/57 [01:39<00:17, 1.40it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 60%|█████▉ | 34/57 [01:39<00:12, 1.83it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 61%|██████▏ | 35/57 [01:39<00:09, 2.36it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 63%|██████▎ | 36/57 [01:39<00:06, 3.02it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 65%|██████▍ | 37/57 [01:39<00:05, 3.78it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 67%|██████▋ | 38/57 [01:39<00:04, 4.61it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 68%|██████▊ | 39/57 [01:39<00:03, 5.47it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 70%|███████ | 40/57 [01:39<00:02, 6.29it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 74%|███████▎ | 42/57 [01:40<00:01, 7.54it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 75%|███████▌ | 43/57 [01:40<00:01, 7.82it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 77%|███████▋ | 44/57 [01:40<00:01, 8.27it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 81%|████████ | 46/57 [01:40<00:01, 8.87it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 82%|████████▏ | 47/57 [01:40<00:01, 9.04it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 84%|████████▍ | 48/57 [01:40<00:00, 9.18it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 86%|████████▌ | 49/57 [01:40<00:00, 9.33it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 88%|████████▊ | 50/57 [01:40<00:00, 9.46it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 89%|████████▉ | 51/57 [01:41<00:00, 9.49it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 91%|█████████ | 52/57 [01:41<00:00, 9.62it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 93%|█████████▎| 53/57 [01:41<00:00, 9.51it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 95%|█████████▍| 54/57 [01:41<00:00, 9.56it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 96%|█████████▋| 55/57 [01:41<00:00, 9.52it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 98%|█████████▊| 56/57 [01:41<00:00, 9.56it/s]Using the latest cached version of the dataset since cais/mmlu couldn't be found on the Hugging Face Hub
|
||
|
||
MMLU subjects: 100%|██████████| 57/57 [01:41<00:00, 1.78s/it]
|
||
[!] Skip high_school_physics: Could not load MMLU subject high_school_physics
|
||
|
||
============================================================
|
||
[>] high_school_psychology (shots=5)
|
||
[!] Skip high_school_psychology: Could not load MMLU subject high_school_psychology
|
||
|
||
============================================================
|
||
[>] high_school_statistics (shots=5)
|
||
[!] Skip high_school_statistics: Could not load MMLU subject high_school_statistics
|
||
|
||
============================================================
|
||
[>] high_school_us_history (shots=5)
|
||
[!] Skip high_school_us_history: Could not load MMLU subject high_school_us_history
|
||
|
||
============================================================
|
||
[>] high_school_world_history (shots=5)
|
||
[!] Skip high_school_world_history: Could not load MMLU subject high_school_world_history
|
||
|
||
============================================================
|
||
[>] human_aging (shots=5)
|
||
[!] Skip human_aging: Could not load MMLU subject human_aging
|
||
|
||
============================================================
|
||
[>] human_sexuality (shots=5)
|
||
[!] Skip human_sexuality: Could not load MMLU subject human_sexuality
|
||
|
||
============================================================
|
||
[>] international_law (shots=5)
|
||
[!] Skip international_law: Could not load MMLU subject international_law
|
||
|
||
============================================================
|
||
[>] jurisprudence (shots=5)
|
||
[!] Skip jurisprudence: Could not load MMLU subject jurisprudence
|
||
|
||
============================================================
|
||
[>] logical_fallacies (shots=5)
|
||
[!] Skip logical_fallacies: Could not load MMLU subject logical_fallacies
|
||
|
||
============================================================
|
||
[>] machine_learning (shots=5)
|
||
[!] Skip machine_learning: Could not load MMLU subject machine_learning
|
||
|
||
============================================================
|
||
[>] management (shots=5)
|
||
[!] Skip management: Could not load MMLU subject management
|
||
|
||
============================================================
|
||
[>] marketing (shots=5)
|
||
[!] Skip marketing: Could not load MMLU subject marketing
|
||
|
||
============================================================
|
||
[>] medical_genetics (shots=5)
|
||
[!] Skip medical_genetics: Could not load MMLU subject medical_genetics
|
||
|
||
============================================================
|
||
[>] miscellaneous (shots=5)
|
||
[!] Skip miscellaneous: Could not load MMLU subject miscellaneous
|
||
|
||
============================================================
|
||
[>] moral_disputes (shots=5)
|
||
[!] Skip moral_disputes: Could not load MMLU subject moral_disputes
|
||
|
||
============================================================
|
||
[>] moral_scenarios (shots=5)
|
||
[!] Skip moral_scenarios: Could not load MMLU subject moral_scenarios
|
||
|
||
============================================================
|
||
[>] nutrition (shots=5)
|
||
[!] Skip nutrition: Could not load MMLU subject nutrition
|
||
|
||
============================================================
|
||
[>] philosophy (shots=5)
|
||
[!] Skip philosophy: Could not load MMLU subject philosophy
|
||
|
||
============================================================
|
||
[>] prehistory (shots=5)
|
||
[!] Skip prehistory: Could not load MMLU subject prehistory
|
||
|
||
============================================================
|
||
[>] professional_accounting (shots=5)
|
||
[!] Skip professional_accounting: Could not load MMLU subject professional_accounting
|
||
|
||
============================================================
|
||
[>] professional_law (shots=5)
|
||
[!] Skip professional_law: Could not load MMLU subject professional_law
|
||
|
||
============================================================
|
||
[>] professional_medicine (shots=5)
|
||
[!] Skip professional_medicine: Could not load MMLU subject professional_medicine
|
||
|
||
============================================================
|
||
[>] professional_psychology (shots=5)
|
||
[!] Skip professional_psychology: Could not load MMLU subject professional_psychology
|
||
|
||
============================================================
|
||
[>] public_relations (shots=5)
|
||
[!] Skip public_relations: Could not load MMLU subject public_relations
|
||
|
||
============================================================
|
||
[>] security_studies (shots=5)
|
||
[!] Skip security_studies: Could not load MMLU subject security_studies
|
||
|
||
============================================================
|
||
[>] sociology (shots=5)
|
||
[!] Skip sociology: Could not load MMLU subject sociology
|
||
|
||
============================================================
|
||
[>] us_foreign_policy (shots=5)
|
||
[!] Skip us_foreign_policy: Could not load MMLU subject us_foreign_policy
|
||
|
||
============================================================
|
||
[>] virology (shots=5)
|
||
[!] Skip virology: Could not load MMLU subject virology
|
||
|
||
============================================================
|
||
[>] world_religions (shots=5)
|
||
[!] Skip world_religions: Could not load MMLU subject world_religions
|
||
|
||
============================================================
|
||
MMLU RESULT — /home/zeus/microllm2/microllm2-checkpoints/final_merged
|
||
Shots: 5 Subjects: 27/57
|
||
abstract_algebra 30.0% (6/20)
|
||
anatomy 25.0% (5/20)
|
||
astronomy 35.0% (7/20)
|
||
business_ethics 30.0% (6/20)
|
||
clinical_knowledge 45.0% (9/20)
|
||
college_biology 45.0% (9/20)
|
||
college_chemistry 15.0% (3/20)
|
||
college_computer_science 45.0% (9/20)
|
||
college_mathematics 35.0% (7/20)
|
||
college_medicine 30.0% (6/20)
|
||
college_physics 15.0% (3/20)
|
||
computer_security 30.0% (6/20)
|
||
conceptual_physics 5.0% (1/20)
|
||
econometrics 30.0% (6/20)
|
||
electrical_engineering 20.0% (4/20)
|
||
elementary_mathematics 30.0% (6/20)
|
||
formal_logic 10.0% (2/20)
|
||
global_facts 35.0% (7/20)
|
||
high_school_biology 45.0% (9/20)
|
||
high_school_chemistry 35.0% (7/20)
|
||
high_school_computer_science 35.0% (7/20)
|
||
high_school_european_history 20.0% (4/20)
|
||
high_school_geography 25.0% (5/20)
|
||
high_school_government_and_politics 20.0% (4/20)
|
||
high_school_macroeconomics 0.0% (0/20)
|
||
high_school_mathematics 20.0% (4/20)
|
||
high_school_microeconomics 35.0% (7/20)
|
||
|
||
OVERALL: 149/540 = 27.59%
|
||
============================================================
|
||
[+] Saved mmlu_results.json
|
||
|
||
Note: GPT2-XL base ~24-26% MMLU (random 25%). MicroLLM2 distilled should be 25-30% —
|
||
MMLU is knowledge-heavy; GPT2 1.5B 1024ctx cannot match 7B+ models. Use as sanity check, not SOTA claim.
|
||
[MMLU] exit 0 at Sun Aug 9 22:53:53 UTC 2026
|
||
"total": 20,
|
||
"accuracy": 0.45
|
||
},
|
||
"college_mathematics": {
|
||
"correct": 7,
|
||
"total": 20,
|
||
"accuracy": 0.35
|
||
},
|
||
"college_medicine": {
|
||
"correct": 6,
|
||
"total": 20,
|
||
"accuracy": 0.3
|
||
},
|
||
"college_physics": {
|
||
"correct": 3,
|
||
"total": 20,
|
||
"accuracy": 0.15
|
||
},
|
||
"computer_security": {
|
||
"correct": 6,
|
||
"total": 20,
|
||
"accuracy": 0.3
|
||
},
|
||
"conceptual_physics": {
|
||
"correct": 1,
|
||
"total": 20,
|
||
"accuracy": 0.05
|
||
},
|
||
"econometrics": {
|
||
"correct": 6,
|
||
"total": 20,
|
||
"accuracy": 0.3
|
||
},
|
||
"electrical_engineering": {
|
||
"correct": 4,
|
||
"total": 20,
|
||
"accuracy": 0.2
|
||
},
|
||
"elementary_mathematics": {
|
||
"correct": 6,
|
||
"total": 20,
|
||
"accuracy": 0.3
|
||
},
|
||
"formal_logic": {
|
||
"correct": 2,
|
||
"total": 20,
|
||
"accuracy": 0.1
|
||
},
|
||
"global_facts": {
|
||
"correct": 7,
|
||
"total": 20,
|
||
"accuracy": 0.35
|
||
},
|
||
"high_school_biology": {
|
||
"correct": 9,
|
||
"total": 20,
|
||
"accuracy": 0.45
|
||
},
|
||
"high_school_chemistry": {
|
||
"correct": 7,
|
||
"total": 20,
|
||
"accuracy": 0.35
|
||
},
|
||
"high_school_computer_science": {
|
||
"correct": 7,
|
||
"total": 20,
|
||
"accuracy": 0.35
|
||
},
|
||
"high_school_european_history": {
|
||
"correct": 4,
|
||
"total": 20,
|
||
"accuracy": 0.2
|
||
},
|
||
"high_school_geography": {
|
||
"correct": 5,
|
||
"total": 20,
|
||
"accuracy": 0.25
|
||
},
|
||
"high_school_government_and_politics": {
|
||
"correct": 4,
|
||
"total": 20,
|
||
"accuracy": 0.2
|
||
},
|
||
"high_school_macroeconomics": {
|
||
"correct": 0,
|
||
"total": 20,
|
||
"accuracy": 0.0
|
||
},
|
||
"high_school_mathematics": {
|
||
"correct": 4,
|
||
"total": 20,
|
||
"accuracy": 0.2
|
||
},
|
||
"high_school_microeconomics": {
|
||
"correct": 7,
|
||
"total": 20,
|
||
"accuracy": 0.35
|
||
}
|
||
}
|
||
}'mmlu_results.json' -> '/teamspace/studios/this_studio/microllm2/mmlu_results.json'
|
||
'mmlu_run.log' -> '/teamspace/studios/this_studio/microllm2/mmlu_run.log'
|