{ "created_at": "2026-07-26T17:00:32.578538+00:00", "benchmark": "BananaMind Base Bench 1.1", "benchmark_version": "1.1", "dataset_id": "BananaMind/BananaMind-Base-Bench-1.1", "dataset_revision": "main", "dataset_file": "/home/appvoid/.cache/huggingface/hub/datasets--BananaMind--BananaMind-Base-Bench-1.1/snapshots/266ef5dc7c812b83c5205235f966ae2f41b3e5b1/data/test.jsonl", "dataset_sha256": "2f563bb46df778ca494fa20f994a8d3045d4c51fbbffeee433764e2813abea21", "model": "/home/appvoid/palmer-006", "model_revision": "main", "tokenizer": "/home/appvoid/palmer-006", "device": "cuda", "dtype": "bfloat16", "model_context_length": 262144, "scoring": { "task": "four-choice base-text continuation", "choice_score": "mean conditional token log-probability", "chat_template": false, "generation": false, "add_special_tokens": false, "add_bos": false }, "elo": { "scale": 400.0, "prior_rating": 1000.0, "prior_weight": 4.0, "method": "weighted fixed-item logistic maximum-likelihood rating" }, "summary": { "official_complete_run": true, "cases": 350, "passed": 231, "accuracy": 0.66, "weighted_points": 425.525, "possible_weighted_points": 668.1875, "weighted_accuracy": 0.6368347207931905, "overall_elo": 1124, "overall_elo_unrounded": 1123.965054004459, "categories": { "language_completion": { "cases": 50, "passed": 47, "accuracy": 0.94, "weighted_points": 73.75, "possible_weighted_points": 78.5, "weighted_accuracy": 0.9394904458598726, "elo": 1345, "elo_unrounded": 1345.2360136505486 }, "commonsense": { "cases": 50, "passed": 35, "accuracy": 0.7, "weighted_points": 58.85, "possible_weighted_points": 87.17500000000001, "weighted_accuracy": 0.6750788643533122, "elo": 1084, "elo_unrounded": 1084.314352251316 }, "world_knowledge": { "cases": 50, "passed": 38, "accuracy": 0.76, "weighted_points": 66.0, "possible_weighted_points": 87.72500000000001, "weighted_accuracy": 0.7523510971786833, "elo": 1151, "elo_unrounded": 1150.5543694613593 }, "context_tracking": { "cases": 50, "passed": 26, "accuracy": 0.52, "weighted_points": 47.699999999999996, "possible_weighted_points": 94.19999999999999, "weighted_accuracy": 0.5063694267515924, "elo": 1002, "elo_unrounded": 1002.410277417023 }, "quantitative": { "cases": 50, "passed": 17, "accuracy": 0.34, "weighted_points": 34.125, "possible_weighted_points": 103.025, "weighted_accuracy": 0.3312302839116719, "elo": 923, "elo_unrounded": 922.9160309715924 }, "logical_reasoning": { "cases": 50, "passed": 25, "accuracy": 0.5, "weighted_points": 51.300000000000004, "possible_weighted_points": 107.66250000000001, "weighted_accuracy": 0.47648902821316613, "elo": 1082, "elo_unrounded": 1081.6919799942607 }, "code_completion": { "cases": 50, "passed": 43, "accuracy": 0.86, "weighted_points": 93.8, "possible_weighted_points": 109.89999999999999, "weighted_accuracy": 0.8535031847133758, "elo": 1403, "elo_unrounded": 1402.753907086344 } }, "difficulties": { "easy": { "cases": 117, "passed": 83, "accuracy": 0.7094017094017094, "weighted_points": 98.60000000000001, "possible_weighted_points": 141.2, "weighted_accuracy": 0.6983002832861192, "elo": 1016, "elo_unrounded": 1016.3133811577516 }, "medium": { "cases": 117, "passed": 77, "accuracy": 0.6581196581196581, "weighted_points": 137.925, "possible_weighted_points": 211.875, "weighted_accuracy": 0.6509734513274337, "elo": 1077, "elo_unrounded": 1076.9044441063706 }, "hard": { "cases": 116, "passed": 71, "accuracy": 0.6120689655172413, "weighted_points": 189.0, "possible_weighted_points": 315.1125, "weighted_accuracy": 0.599785790789004, "elo": 1187, "elo_unrounded": 1186.5309969349169 } }, "context_truncations": 0 }, "results": [ { "id": "language_completion-001", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "Dark clouds gathered above the park, so Mira opened", "continuations": [ " a sandwich loudly.", " the sunlight.", " her umbrella.", " the calendar yesterday." ], "tags": [ "coherence" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -29.258682250976562, "mean_logprob": -4.876447041829427, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 3, "sum_logprob": -14.473627090454102, "mean_logprob": -4.8245423634847, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 6, "sum_logprob": -8.218729019165039, "mean_logprob": -1.3697881698608398, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 4, "sum_logprob": -22.170177459716797, "mean_logprob": -5.542544364929199, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 3.4547541936238604, "context_truncated": false, "elapsed_seconds": 0.9227663130004657 }, { "id": "language_completion-002", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "The librarian asked everyone to speak", "continuations": [ " brightly.", " quietly.", " woodenly.", " tomorrow." ], "tags": [ "adverb" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -15.653277397155762, "mean_logprob": -5.217759132385254, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 3, "sum_logprob": -7.187465190887451, "mean_logprob": -2.395821730295817, "total_sequence_tokens": 11 }, { "index": 2, "tokens": 3, "sum_logprob": -20.17020034790039, "mean_logprob": -6.723400115966797, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 2, "sum_logprob": -11.879611015319824, "mean_logprob": -5.939805507659912, "total_sequence_tokens": 10 } ], "best_vs_second_margin": 2.821937402089437, "context_truncated": false, "elapsed_seconds": 0.051079696999295265 }, { "id": "language_completion-003", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 0.0, "item_elo": 750, "context": "The brass key fit perfectly into the", "continuations": [ " pillow.", " thunder.", " lock.", " soup." ], "tags": [ "collocation" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -10.946988105773926, "mean_logprob": -3.648996035257975, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 3, "sum_logprob": -18.382553100585938, "mean_logprob": -6.1275177001953125, "total_sequence_tokens": 11 }, { "index": 2, "tokens": 2, "sum_logprob": -7.96212100982666, "mean_logprob": -3.98106050491333, "total_sequence_tokens": 10 }, { "index": 3, "tokens": 2, "sum_logprob": -14.014673233032227, "mean_logprob": -7.007336616516113, "total_sequence_tokens": 10 } ], "best_vs_second_margin": 0.33206446965535497, "context_truncated": false, "elapsed_seconds": 0.043018684000344365 }, { "id": "language_completion-004", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "After kneading the dough, the baker placed it in the", "continuations": [ " envelope.", " freezer to read.", " bicycle.", " oven." ], "tags": [ "coherence" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -16.552427291870117, "mean_logprob": -8.276213645935059, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 5, "sum_logprob": -25.37650489807129, "mean_logprob": -5.075300979614258, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 2, "sum_logprob": -25.005922317504883, "mean_logprob": -12.502961158752441, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 2, "sum_logprob": -2.8407983779907227, "mean_logprob": -1.4203991889953613, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 3.6549017906188963, "context_truncated": false, "elapsed_seconds": 0.05098429600002419 }, { "id": "language_completion-005", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "Leo was exhausted after the hike and went straight to", "continuations": [ " bed.", " the trumpet.", " blue.", " calculate." ], "tags": [ "coherence" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -2.9319279193878174, "mean_logprob": -1.4659639596939087, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 5, "sum_logprob": -15.567556381225586, "mean_logprob": -3.1135112762451174, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 2, "sum_logprob": -10.717717170715332, "mean_logprob": -5.358858585357666, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 2, "sum_logprob": -17.180051803588867, "mean_logprob": -8.590025901794434, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 1.6475473165512087, "context_truncated": false, "elapsed_seconds": 0.041541919999872334 }, { "id": "language_completion-006", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "The ice cube disappeared because it had", "continuations": [ " flown north.", " melted.", " written a letter.", " become louder." ], "tags": [ "verb" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -15.799678802490234, "mean_logprob": -3.9499197006225586, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 3, "sum_logprob": -3.6080100536346436, "mean_logprob": -1.2026700178782146, "total_sequence_tokens": 10 }, { "index": 2, "tokens": 4, "sum_logprob": -17.65917205810547, "mean_logprob": -4.414793014526367, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 5, "sum_logprob": -17.602617263793945, "mean_logprob": -3.520523452758789, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 2.3178534348805746, "context_truncated": false, "elapsed_seconds": 0.0459696409998287 }, { "id": "language_completion-007", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "One child waited outside; soon, three more", "continuations": [ " childs arrived.", " childes arrived.", " children arrived.", " child arriveds." ], "tags": [ "grammar" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -14.999372482299805, "mean_logprob": -3.749843120574951, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 4, "sum_logprob": -24.83102798461914, "mean_logprob": -6.207756996154785, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 3, "sum_logprob": -6.094472885131836, "mean_logprob": -2.031490961710612, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 4, "sum_logprob": -26.04423713684082, "mean_logprob": -6.511059284210205, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 1.7183521588643393, "context_truncated": false, "elapsed_seconds": 0.04569892300060019 }, { "id": "language_completion-008", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "Yesterday, Priya walked to the station and", "continuations": [ " waits for tomorrow.", " waiting there now.", " will walked home.", " bought a ticket." ], "tags": [ "tense" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -27.789226531982422, "mean_logprob": -5.5578453063964846, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 4, "sum_logprob": -24.613021850585938, "mean_logprob": -6.153255462646484, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 4, "sum_logprob": -27.470468521118164, "mean_logprob": -6.867617130279541, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 4, "sum_logprob": -9.572569847106934, "mean_logprob": -2.3931424617767334, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 3.164702844619751, "context_truncated": false, "elapsed_seconds": 0.043610005001028185 }, { "id": "language_completion-009", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "The cup is empty, but the pitcher is", "continuations": [ " full.", " emptier than silence.", " running yesterday.", " carefully." ], "tags": [ "contrast" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -0.5352787971496582, "mean_logprob": -0.2676393985748291, "total_sequence_tokens": 12 }, { "index": 1, "tokens": 6, "sum_logprob": -30.967300415039062, "mean_logprob": -5.161216735839844, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 3, "sum_logprob": -23.753543853759766, "mean_logprob": -7.917847951253255, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 2, "sum_logprob": -22.881458282470703, "mean_logprob": -11.440729141235352, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 4.893577337265015, "context_truncated": false, "elapsed_seconds": 0.04228099199826829 }, { "id": "language_completion-010", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "Neither the teacher nor the students", "continuations": [ " was arrives.", " were ready.", " is readiness.", " be readying." ], "tags": [ "agreement" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -25.68792152404785, "mean_logprob": -6.421980381011963, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 3, "sum_logprob": -9.509971618652344, "mean_logprob": -3.1699905395507812, "total_sequence_tokens": 10 }, { "index": 2, "tokens": 4, "sum_logprob": -28.661731719970703, "mean_logprob": -7.165432929992676, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 4, "sum_logprob": -28.500198364257812, "mean_logprob": -7.125049591064453, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 3.2519898414611816, "context_truncated": false, "elapsed_seconds": 0.05342107900105475 }, { "id": "language_completion-011", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "Nora packed an apple, a bottle of water, and", "continuations": [ " because.", " beneath.", " a sandwich.", " although." ], "tags": [ "parallelism" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -21.972896575927734, "mean_logprob": -10.986448287963867, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 2, "sum_logprob": -21.476001739501953, "mean_logprob": -10.738000869750977, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 4, "sum_logprob": -7.899407386779785, "mean_logprob": -1.9748518466949463, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 2, "sum_logprob": -24.667869567871094, "mean_logprob": -12.333934783935547, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 8.76314902305603, "context_truncated": false, "elapsed_seconds": 0.03638890000001993 }, { "id": "language_completion-012", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "A robin is a bird, and a salmon is", "continuations": [ " beneath flying.", " an quickly.", " yesterday's wing.", " a fish." ], "tags": [ "syntax" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -33.78597640991211, "mean_logprob": -11.26199213663737, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 3, "sum_logprob": -33.265933990478516, "mean_logprob": -11.088644663492838, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 5, "sum_logprob": -36.72492980957031, "mean_logprob": -7.3449859619140625, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 3, "sum_logprob": -0.36596569418907166, "mean_logprob": -0.12198856472969055, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 7.222997397184372, "context_truncated": false, "elapsed_seconds": 0.03459843299970089 }, { "id": "language_completion-013", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "She adopted a puppy from", "continuations": [ " an animal shelter.", " shelter animal an.", " a sheltering quickly.", " the adopted because." ], "tags": [ "article" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -7.865048408508301, "mean_logprob": -1.9662621021270752, "total_sequence_tokens": 10 }, { "index": 1, "tokens": 4, "sum_logprob": -32.348663330078125, "mean_logprob": -8.087165832519531, "total_sequence_tokens": 10 }, { "index": 2, "tokens": 5, "sum_logprob": -33.793785095214844, "mean_logprob": -6.758757019042969, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 4, "sum_logprob": -30.407835006713867, "mean_logprob": -7.601958751678467, "total_sequence_tokens": 10 } ], "best_vs_second_margin": 4.792494916915894, "context_truncated": false, "elapsed_seconds": 0.03482184299900837 }, { "id": "language_completion-014", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "Daniel thanked Rosa because", "continuations": [ " him had helped she.", " she had helped him.", " her helped himself.", " they helps he." ], "tags": [ "pronouns" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -32.18361282348633, "mean_logprob": -6.4367225646972654, "total_sequence_tokens": 12 }, { "index": 1, "tokens": 5, "sum_logprob": -11.391644477844238, "mean_logprob": -2.2783288955688477, "total_sequence_tokens": 12 }, { "index": 2, "tokens": 4, "sum_logprob": -28.635631561279297, "mean_logprob": -7.158907890319824, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 4, "sum_logprob": -32.779876708984375, "mean_logprob": -8.194969177246094, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 4.158393669128418, "context_truncated": false, "elapsed_seconds": 0.03904251700078021 }, { "id": "language_completion-015", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "This suitcase is heavy, but that one is even", "continuations": [ " most heavy.", " heavily one.", " heavier.", " more heavier." ], "tags": [ "comparative" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -16.571857452392578, "mean_logprob": -5.523952484130859, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 3, "sum_logprob": -23.978748321533203, "mean_logprob": -7.992916107177734, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 3, "sum_logprob": -1.3102918863296509, "mean_logprob": -0.4367639621098836, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 4, "sum_logprob": -8.343090057373047, "mean_logprob": -2.0857725143432617, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 1.6490085522333782, "context_truncated": false, "elapsed_seconds": 0.03316458200060879 }, { "id": "language_completion-016", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "Monday comes immediately after", "continuations": [ " Wednesday.", " evening.", " January.", " Sunday." ], "tags": [ "sequence" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -9.539777755737305, "mean_logprob": -4.769888877868652, "total_sequence_tokens": 7 }, { "index": 1, "tokens": 2, "sum_logprob": -11.153751373291016, "mean_logprob": -5.576875686645508, "total_sequence_tokens": 7 }, { "index": 2, "tokens": 2, "sum_logprob": -11.202709197998047, "mean_logprob": -5.601354598999023, "total_sequence_tokens": 7 }, { "index": 3, "tokens": 2, "sum_logprob": -7.936887264251709, "mean_logprob": -3.9684436321258545, "total_sequence_tokens": 7 } ], "best_vs_second_margin": 0.8014452457427979, "context_truncated": false, "elapsed_seconds": 0.04127686799984076 }, { "id": "language_completion-017", "category": "language_completion", "difficulty": "easy", "category_weight": 1.0, "difficulty_weight": 1.0, "point_weight": 1.0, "earned_points": 1.0, "item_elo": 750, "context": "Although the road was narrow, the driver remained", "continuations": [ " careful.", " width.", " carefully road.", " narrowness." ], "tags": [ "grammar" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -7.293829917907715, "mean_logprob": -3.6469149589538574, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 2, "sum_logprob": -20.16122055053711, "mean_logprob": -10.080610275268555, "total_sequence_tokens": 11 }, { "index": 2, "tokens": 3, "sum_logprob": -23.18958854675293, "mean_logprob": -7.7298628489176435, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 3, "sum_logprob": -20.18323516845703, "mean_logprob": -6.727745056152344, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 3.0808300971984863, "context_truncated": false, "elapsed_seconds": 0.033489133000330185 }, { "id": "language_completion-018", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 0.0, "item_elo": 850, "context": "The forecast promised sunshine. Nevertheless, we packed", "continuations": [ " the forecast itself.", " rain jackets.", " a darker noon.", " sunshine into boxes." ], "tags": [ "discourse" ], "prediction": 3, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -19.17872428894043, "mean_logprob": -4.794681072235107, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 4, "sum_logprob": -16.765783309936523, "mean_logprob": -4.191445827484131, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 6, "sum_logprob": -29.223602294921875, "mean_logprob": -4.8706003824869795, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 6, "sum_logprob": -22.123672485351562, "mean_logprob": -3.6872787475585938, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 0.5041670799255371, "context_truncated": false, "elapsed_seconds": 0.03606204700008675 }, { "id": "language_completion-019", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "Amir revised the report twice before submitting it, thereby", "continuations": [ " forgetting what reports are.", " submitting before revising.", " reducing the chance of errors.", " making twice become three." ], "tags": [ "discourse" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -40.61179733276367, "mean_logprob": -6.768632888793945, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 6, "sum_logprob": -27.6369686126709, "mean_logprob": -4.606161435445149, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 6, "sum_logprob": -12.523467063903809, "mean_logprob": -2.0872445106506348, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 5, "sum_logprob": -44.723777770996094, "mean_logprob": -8.944755554199219, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 2.5189169247945147, "context_truncated": false, "elapsed_seconds": 0.034903477999250754 }, { "id": "language_completion-020", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "The museum closes at six. We arrived at five forty-five, leaving us", "continuations": [ " closed before arrival.", " six museums to visit.", " yesterday instead.", " only fifteen minutes." ], "tags": [ "coherence" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -27.271808624267578, "mean_logprob": -6.8179521560668945, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 5, "sum_logprob": -19.100353240966797, "mean_logprob": -3.8200706481933593, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 3, "sum_logprob": -22.578886032104492, "mean_logprob": -7.526295344034831, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 4, "sum_logprob": -13.414216995239258, "mean_logprob": -3.3535542488098145, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 0.46651639938354483, "context_truncated": false, "elapsed_seconds": 0.03929743299886468 }, { "id": "language_completion-021", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "The committee postponed the vote until every member had", "continuations": [ " reviewed the proposal.", " been proposal by voting.", " postponed every committee.", " review the voted." ], "tags": [ "aspect" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -10.851517677307129, "mean_logprob": -2.7128794193267822, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 5, "sum_logprob": -34.83161544799805, "mean_logprob": -6.9663230895996096, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 6, "sum_logprob": -32.971126556396484, "mean_logprob": -5.495187759399414, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 4, "sum_logprob": -32.07133483886719, "mean_logprob": -8.017833709716797, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 2.782308340072632, "context_truncated": false, "elapsed_seconds": 0.030963249000706128 }, { "id": "language_completion-022", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "Had Elena known the bridge was closed, she", "continuations": [ " takes that route anyway.", " would have taken another route.", " will know it tomorrow yesterday.", " had closes the bridge." ], "tags": [ "conditional" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -27.9699649810791, "mean_logprob": -5.59399299621582, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 6, "sum_logprob": -18.3637752532959, "mean_logprob": -3.06062920888265, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 6, "sum_logprob": -39.02067184448242, "mean_logprob": -6.503445307413737, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 6, "sum_logprob": -29.636816024780273, "mean_logprob": -4.939469337463379, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 1.878840128580729, "context_truncated": false, "elapsed_seconds": 0.03254597900013323 }, { "id": "language_completion-023", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "The novel was praised not only for its plot but also for", "continuations": [ " it was long.", " plotting beyond.", " its vivid characters.", " praise was also." ], "tags": [ "parallelism" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -33.65026092529297, "mean_logprob": -8.412565231323242, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 4, "sum_logprob": -31.721717834472656, "mean_logprob": -7.930429458618164, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 4, "sum_logprob": -13.388650894165039, "mean_logprob": -3.3471627235412598, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 5, "sum_logprob": -50.499691009521484, "mean_logprob": -10.099938201904298, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 4.583266735076904, "context_truncated": false, "elapsed_seconds": 0.0373120789990935 }, { "id": "language_completion-024", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "Because the instructions were ambiguous, two technicians interpreted them", "continuations": [ " with identical ambiguity only.", " before they were written.", " into a screwdriver.", " differently." ], "tags": [ "semantics" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -30.603696823120117, "mean_logprob": -6.120739364624024, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 5, "sum_logprob": -17.289165496826172, "mean_logprob": -3.457833099365234, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 6, "sum_logprob": -19.69405174255371, "mean_logprob": -3.282341957092285, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 2, "sum_logprob": -2.138988971710205, "mean_logprob": -1.0694944858551025, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 2.2128474712371826, "context_truncated": false, "elapsed_seconds": 0.02990865999890957 }, { "id": "language_completion-025", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "Sofia lowered her voice so that she would not", "continuations": [ " wake the baby.", " sleeping the quiet.", " lower the voice's color.", " have loudly whisper." ], "tags": [ "purpose" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -16.679588317871094, "mean_logprob": -4.169897079467773, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 4, "sum_logprob": -38.57061767578125, "mean_logprob": -9.642654418945312, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 7, "sum_logprob": -33.39238739013672, "mean_logprob": -4.770341055733817, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 6, "sum_logprob": -32.384063720703125, "mean_logprob": -5.3973439534505205, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 0.6004439762660434, "context_truncated": false, "elapsed_seconds": 0.03863776300022437 }, { "id": "language_completion-026", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "The first experiment failed; the second, by contrast,", "continuations": [ " was experiment first.", " produced reliable results.", " contrasted before being second.", " failing reliably." ], "tags": [ "discourse" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -27.998268127441406, "mean_logprob": -6.999567031860352, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 4, "sum_logprob": -9.488460540771484, "mean_logprob": -2.372115135192871, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 6, "sum_logprob": -40.71430206298828, "mean_logprob": -6.785717010498047, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 4, "sum_logprob": -21.225833892822266, "mean_logprob": -5.306458473205566, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 2.9343433380126953, "context_truncated": false, "elapsed_seconds": 0.028666005999184563 }, { "id": "language_completion-027", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "Every applicant must attach a resume, regardless of", "continuations": [ " attaching is a verb.", " resumes being paper.", " prior experience.", " every regardless." ], "tags": [ "collocation" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -33.530799865722656, "mean_logprob": -5.588466644287109, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 5, "sum_logprob": -28.748760223388672, "mean_logprob": -5.749752044677734, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 3, "sum_logprob": -9.654231071472168, "mean_logprob": -3.218077023824056, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 3, "sum_logprob": -31.807788848876953, "mean_logprob": -10.602596282958984, "total_sequence_tokens": 13 } ], "best_vs_second_margin": 2.3703896204630532, "context_truncated": false, "elapsed_seconds": 0.02823326300131157 }, { "id": "language_completion-028", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "The recipe calls for fresh basil; dried basil may be used", "continuations": [ " before fresh basil exists.", " to call the recipe.", " as a plate.", " as a substitute." ], "tags": [ "coherence" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -19.658309936523438, "mean_logprob": -3.2763849894205728, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 5, "sum_logprob": -22.95904541015625, "mean_logprob": -4.59180908203125, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 4, "sum_logprob": -20.08816146850586, "mean_logprob": -5.022040367126465, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 4, "sum_logprob": -4.103679180145264, "mean_logprob": -1.025919795036316, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 2.250465194384257, "context_truncated": false, "elapsed_seconds": 0.03386163600043801 }, { "id": "language_completion-029", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "Far from being discouraged by the criticism, Mei became", "continuations": [ " more determined.", " criticism itself.", " far from becoming.", " determine yesterday." ], "tags": [ "idiom" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -8.815675735473633, "mean_logprob": -2.938558578491211, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 3, "sum_logprob": -23.545949935913086, "mean_logprob": -7.848649978637695, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 4, "sum_logprob": -30.538150787353516, "mean_logprob": -7.634537696838379, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 3, "sum_logprob": -30.116004943847656, "mean_logprob": -10.038668314615885, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 4.695979118347168, "context_truncated": false, "elapsed_seconds": 0.02743094600009499 }, { "id": "language_completion-030", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "The contract is valid provided that both parties", "continuations": [ " had validity tomorrow.", " sign it.", " providing a party.", " signs them." ], "tags": [ "conditional" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -35.413047790527344, "mean_logprob": -8.853261947631836, "total_sequence_tokens": 12 }, { "index": 1, "tokens": 3, "sum_logprob": -8.452535629272461, "mean_logprob": -2.817511876424154, "total_sequence_tokens": 11 }, { "index": 2, "tokens": 4, "sum_logprob": -28.993125915527344, "mean_logprob": -7.248281478881836, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 3, "sum_logprob": -19.822309494018555, "mean_logprob": -6.607436498006185, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 3.789924621582031, "context_truncated": false, "elapsed_seconds": 0.028958492999663576 }, { "id": "language_completion-031", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "No sooner had the bell rung than the students", "continuations": [ " have ringing bells.", " sooner the bell.", " left the classroom.", " had leave tomorrow." ], "tags": [ "inversion" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -27.051218032836914, "mean_logprob": -4.508536338806152, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 5, "sum_logprob": -22.70492935180664, "mean_logprob": -4.540985870361328, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 4, "sum_logprob": -8.72513484954834, "mean_logprob": -2.181283712387085, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 4, "sum_logprob": -21.85160255432129, "mean_logprob": -5.462900638580322, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 2.3272526264190674, "context_truncated": false, "elapsed_seconds": 0.03914885999984108 }, { "id": "language_completion-032", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "The evidence was incomplete. Consequently, the investigators", "continuations": [ " completed evidence by definition.", " investigated before consequence.", " were the evidence.", " withheld judgment." ], "tags": [ "discourse" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -37.00300598144531, "mean_logprob": -7.400601196289062, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 4, "sum_logprob": -32.91265869140625, "mean_logprob": -8.228164672851562, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 4, "sum_logprob": -31.54528045654297, "mean_logprob": -7.886320114135742, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 4, "sum_logprob": -17.593650817871094, "mean_logprob": -4.398412704467773, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 3.002188491821289, "context_truncated": false, "elapsed_seconds": 0.033153643998957705 }, { "id": "language_completion-033", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "Walking along the shore, I noticed", "continuations": [ " a trail of fresh footprints.", " the shore walked me.", " noticing had walked.", " along was a footprint." ], "tags": [ "modifier" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -24.315420150756836, "mean_logprob": -3.4736314501081194, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 5, "sum_logprob": -27.378658294677734, "mean_logprob": -5.475731658935547, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 5, "sum_logprob": -39.87729263305664, "mean_logprob": -7.9754585266113285, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 6, "sum_logprob": -41.18281936645508, "mean_logprob": -6.863803227742513, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 2.0021002088274273, "context_truncated": false, "elapsed_seconds": 0.03778556900033436 }, { "id": "language_completion-034", "category": "language_completion", "difficulty": "medium", "category_weight": 1.0, "difficulty_weight": 1.5, "point_weight": 1.5, "earned_points": 1.5, "item_elo": 850, "context": "The orchestra rehearsed the difficult passage repeatedly until it sounded", "continuations": [ " passage by difficulty.", " effortless.", " rehearsing the sound.", " repeatably orchestra." ], "tags": [ "collocation" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -32.8645133972168, "mean_logprob": -8.2161283493042, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 3, "sum_logprob": -5.816623687744141, "mean_logprob": -1.9388745625813801, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 7, "sum_logprob": -31.051162719726562, "mean_logprob": -4.435880388532366, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 6, "sum_logprob": -39.944427490234375, "mean_logprob": -6.6574045817057295, "total_sequence_tokens": 22 } ], "best_vs_second_margin": 2.4970058259509864, "context_truncated": false, "elapsed_seconds": 0.04088619899994228 }, { "id": "language_completion-035", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "What the witness omitted from her statement proved", "continuations": [ " that omission witnessed.", " proving had omitted.", " more significant than what she included.", " to include more omittedly." ], "tags": [ "syntax" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -31.159378051757812, "mean_logprob": -6.231875610351563, "total_sequence_tokens": 13 }, { "index": 1, "tokens": 4, "sum_logprob": -33.38984680175781, "mean_logprob": -8.347461700439453, "total_sequence_tokens": 12 }, { "index": 2, "tokens": 7, "sum_logprob": -23.858367919921875, "mean_logprob": -3.4083382742745534, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 6, "sum_logprob": -43.83302307128906, "mean_logprob": -7.305503845214844, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 2.8235373360770093, "context_truncated": false, "elapsed_seconds": 0.05545987199911906 }, { "id": "language_completion-036", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "Were the assumptions underlying the estimate to change, the conclusion would", "continuations": [ " have assumptions beneath it.", " changing was estimated.", " underlie tomorrow.", " need to be reconsidered." ], "tags": [ "conditional" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -28.43362045288086, "mean_logprob": -5.686724090576172, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 4, "sum_logprob": -36.489532470703125, "mean_logprob": -9.122383117675781, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 4, "sum_logprob": -29.69447135925293, "mean_logprob": -7.423617839813232, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 7, "sum_logprob": -9.104352951049805, "mean_logprob": -1.300621850149972, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 4.3861022404262, "context_truncated": false, "elapsed_seconds": 0.029180166000514873 }, { "id": "language_completion-037", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "The policy, while defensible in principle, is difficult to", "continuations": [ " implement consistently.", " principle the defense.", " difficult implementation was.", " consistency defending." ], "tags": [ "concession" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -6.661409378051758, "mean_logprob": -2.2204697926839194, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 4, "sum_logprob": -35.245487213134766, "mean_logprob": -8.811371803283691, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 4, "sum_logprob": -46.78346633911133, "mean_logprob": -11.695866584777832, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 4, "sum_logprob": -36.44639587402344, "mean_logprob": -9.11159896850586, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 6.5909020105997715, "context_truncated": false, "elapsed_seconds": 0.027174740000191377 }, { "id": "language_completion-038", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "Only after the archive was digitized did researchers", "continuations": [ " digitized only before.", " discover the missing correspondence.", " archive the discovery did.", " correspondingly research." ], "tags": [ "inversion" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -35.61037826538086, "mean_logprob": -7.1220756530761715, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 5, "sum_logprob": -25.180889129638672, "mean_logprob": -5.0361778259277346, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 5, "sum_logprob": -38.086387634277344, "mean_logprob": -7.617277526855469, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 4, "sum_logprob": -29.758289337158203, "mean_logprob": -7.439572334289551, "total_sequence_tokens": 13 } ], "best_vs_second_margin": 2.085897827148437, "context_truncated": false, "elapsed_seconds": 0.030827698999928543 }, { "id": "language_completion-039", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "The editor objected less to the claim itself than to", "continuations": [ " claims having objections.", " itself being less.", " the certainty with which it was stated.", " stating an editor." ], "tags": [ "comparison" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -23.438323974609375, "mean_logprob": -3.9063873291015625, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 4, "sum_logprob": -33.21238708496094, "mean_logprob": -8.303096771240234, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 9, "sum_logprob": -24.74079132080078, "mean_logprob": -2.748976813422309, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 5, "sum_logprob": -31.99425506591797, "mean_logprob": -6.398851013183593, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 1.1574105156792536, "context_truncated": false, "elapsed_seconds": 0.026268130000971723 }, { "id": "language_completion-040", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "So subtle was the change in tone that few listeners", "continuations": [ " made tone into change.", " subtly were few.", " had listened tomorrow.", " noticed it." ], "tags": [ "inversion" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -36.30894470214844, "mean_logprob": -7.261788940429687, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 5, "sum_logprob": -43.557823181152344, "mean_logprob": -8.711564636230468, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 5, "sum_logprob": -30.78145980834961, "mean_logprob": -6.156291961669922, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 3, "sum_logprob": -13.248579025268555, "mean_logprob": -4.416193008422852, "total_sequence_tokens": 13 } ], "best_vs_second_margin": 1.7400989532470703, "context_truncated": false, "elapsed_seconds": 0.026592488000460435 }, { "id": "language_completion-041", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "The proposal is neither as costly as its critics allege nor as simple as its supporters", "continuations": [ " suggest.", " allegation was cost.", " supporting simply.", " critics and costly." ], "tags": [ "parallelism" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -4.625497817993164, "mean_logprob": -2.312748908996582, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 5, "sum_logprob": -36.28232192993164, "mean_logprob": -7.2564643859863285, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 3, "sum_logprob": -27.830577850341797, "mean_logprob": -9.276859283447266, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 4, "sum_logprob": -37.068687438964844, "mean_logprob": -9.267171859741211, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 4.9437154769897464, "context_truncated": false, "elapsed_seconds": 0.0277293990002363 }, { "id": "language_completion-042", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "Whatever reservations the panel may have had, they were not sufficient to", "continuations": [ " reserve a panel seat.", " block the appointment.", " appoint sufficient reservations.", " having been whatever." ], "tags": [ "concession" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -20.758148193359375, "mean_logprob": -4.151629638671875, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 4, "sum_logprob": -14.865694046020508, "mean_logprob": -3.716423511505127, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 5, "sum_logprob": -24.439010620117188, "mean_logprob": -4.8878021240234375, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 4, "sum_logprob": -34.549068450927734, "mean_logprob": -8.637267112731934, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.4352061271667482, "context_truncated": false, "elapsed_seconds": 0.05680329200004053 }, { "id": "language_completion-043", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "Had the data been interpreted in isolation, the apparent trend might have", "continuations": [ " isolated its data.", " been interpreting apparent.", " seemed more convincing.", " trended the isolation." ], "tags": [ "counterfactual" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -24.237958908081055, "mean_logprob": -6.059489727020264, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 5, "sum_logprob": -35.368247985839844, "mean_logprob": -7.073649597167969, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 6, "sum_logprob": -17.2215518951416, "mean_logprob": -2.870258649190267, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 5, "sum_logprob": -35.14228820800781, "mean_logprob": -7.0284576416015625, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 3.1892310778299966, "context_truncated": false, "elapsed_seconds": 0.059677686000213725 }, { "id": "language_completion-044", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "The mayor's apology did little to allay concerns that the inquiry had been", "continuations": [ " mayoral and apologetic.", " concerned into inquiry.", " doing little apology.", " deliberately delayed." ], "tags": [ "collocation" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -28.935810089111328, "mean_logprob": -4.1336871555873325, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 4, "sum_logprob": -41.75049591064453, "mean_logprob": -10.437623977661133, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 5, "sum_logprob": -31.012474060058594, "mean_logprob": -6.202494812011719, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 4, "sum_logprob": -6.5513129234313965, "mean_logprob": -1.6378282308578491, "total_sequence_tokens": 22 } ], "best_vs_second_margin": 2.4958589247294833, "context_truncated": false, "elapsed_seconds": 0.04653508400042483 }, { "id": "language_completion-045", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "Not until the final chapter does the significance of the opening scene become", "continuations": [ " fully apparent.", " final before opening.", " chaptered significantly.", " opening does final." ], "tags": [ "inversion" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -7.648927211761475, "mean_logprob": -2.5496424039204917, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 4, "sum_logprob": -25.284252166748047, "mean_logprob": -6.321063041687012, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 4, "sum_logprob": -37.81638717651367, "mean_logprob": -9.454096794128418, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 4, "sum_logprob": -36.81341552734375, "mean_logprob": -9.203353881835938, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 3.77142063776652, "context_truncated": false, "elapsed_seconds": 0.027118156998767518 }, { "id": "language_completion-046", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 0.0, "item_elo": 1000, "context": "The theory's appeal lies not in its novelty but in its ability to", "continuations": [ " appeal theoretically.", " unify previously separate observations.", " become novel before theory.", " observe its ability." ], "tags": [ "semantics" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -18.210508346557617, "mean_logprob": -3.6421016693115233, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 6, "sum_logprob": -25.781158447265625, "mean_logprob": -4.2968597412109375, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 5, "sum_logprob": -40.01434326171875, "mean_logprob": -8.00286865234375, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 4, "sum_logprob": -31.227251052856445, "mean_logprob": -7.806812763214111, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.6547580718994142, "context_truncated": false, "elapsed_seconds": 0.03853663200061419 }, { "id": "language_completion-047", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "Much as the negotiators wanted an agreement, neither side was prepared to", "continuations": [ " negotiate wanting.", " agreement much as.", " concede the central point.", " prepare neither side." ], "tags": [ "concession" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -23.181316375732422, "mean_logprob": -4.636263275146485, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 4, "sum_logprob": -35.157752990722656, "mean_logprob": -8.789438247680664, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 7, "sum_logprob": -22.38667106628418, "mean_logprob": -3.1980958666120256, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 4, "sum_logprob": -27.910926818847656, "mean_logprob": -6.977731704711914, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 1.4381674085344591, "context_truncated": false, "elapsed_seconds": 0.027152167000167537 }, { "id": "language_completion-048", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "Whether the discrepancy reflects measurement error or a genuine anomaly remains", "continuations": [ " measuring a reflection.", " discrepant in error.", " anomalously genuine.", " uncertain." ], "tags": [ "syntax" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -34.6393928527832, "mean_logprob": -8.6598482131958, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 6, "sum_logprob": -32.52997970581055, "mean_logprob": -5.421663284301758, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 5, "sum_logprob": -37.232521057128906, "mean_logprob": -7.446504211425781, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 2, "sum_logprob": -2.806727170944214, "mean_logprob": -1.403363585472107, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 4.018299698829651, "context_truncated": false, "elapsed_seconds": 0.02666539100027876 }, { "id": "language_completion-049", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "The ruling established a precedent whose consequences would not become clear", "continuations": [ " for several years.", " precedent had ruled.", " consequences establishing.", " clearly whose." ], "tags": [ "relative_clause" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -9.659148216247559, "mean_logprob": -2.4147870540618896, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 5, "sum_logprob": -32.767478942871094, "mean_logprob": -6.553495788574219, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 3, "sum_logprob": -32.46292495727539, "mean_logprob": -10.820974985758463, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 3, "sum_logprob": -32.78173828125, "mean_logprob": -10.92724609375, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 4.1387087345123295, "context_truncated": false, "elapsed_seconds": 0.026745708999442286 }, { "id": "language_completion-050", "category": "language_completion", "difficulty": "hard", "category_weight": 1.0, "difficulty_weight": 2.25, "point_weight": 2.25, "earned_points": 2.25, "item_elo": 1000, "context": "To attribute the recovery solely to the new policy would be to", "continuations": [ " recover sole attribution.", " ignore several contributing factors.", " policy the recovery solely.", " contribute an attribution." ], "tags": [ "idiom" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -28.278736114501953, "mean_logprob": -5.6557472229003904, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 5, "sum_logprob": -23.966026306152344, "mean_logprob": -4.793205261230469, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 5, "sum_logprob": -29.296300888061523, "mean_logprob": -5.859260177612304, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 5, "sum_logprob": -25.130556106567383, "mean_logprob": -5.026111221313476, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 0.23290596008300746, "context_truncated": false, "elapsed_seconds": 0.027140629999848898 }, { "id": "commonsense-001", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "A glass slipped from the table onto a tile floor. It most likely", "continuations": [ " grew roots.", " floated upward.", " shattered.", " became warmer than fire." ], "tags": [ "physical" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -17.35636329650879, "mean_logprob": -5.785454432169597, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 4, "sum_logprob": -17.766342163085938, "mean_logprob": -4.441585540771484, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 4, "sum_logprob": -8.128620147705078, "mean_logprob": -2.0321550369262695, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 5, "sum_logprob": -26.523273468017578, "mean_logprob": -5.304654693603515, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 2.409430503845215, "context_truncated": false, "elapsed_seconds": 0.07991852200029825 }, { "id": "commonsense-002", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "Jules wanted to mail a letter, so he put a stamp on the", "continuations": [ " refrigerator.", " envelope.", " sidewalk.", " spoon." ], "tags": [ "everyday" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -10.811249732971191, "mean_logprob": -2.702812433242798, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 2, "sum_logprob": -0.9752203226089478, "mean_logprob": -0.4876101613044739, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 4, "sum_logprob": -11.075915336608887, "mean_logprob": -2.7689788341522217, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 3, "sum_logprob": -11.012920379638672, "mean_logprob": -3.670973459879557, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 2.215202271938324, "context_truncated": false, "elapsed_seconds": 0.0346436170002562 }, { "id": "commonsense-003", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "The room was dark until Ana flipped the wall switch and the", "continuations": [ " carpet vanished.", " window froze.", " lights came on.", " ceiling became wet." ], "tags": [ "causality" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -19.102373123168945, "mean_logprob": -3.820474624633789, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 4, "sum_logprob": -11.570735931396484, "mean_logprob": -2.892683982849121, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 4, "sum_logprob": -7.960273265838623, "mean_logprob": -1.9900683164596558, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 4, "sum_logprob": -17.106468200683594, "mean_logprob": -4.276617050170898, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 0.9026156663894653, "context_truncated": false, "elapsed_seconds": 0.028131573999417014 }, { "id": "commonsense-004", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "Before slicing a loaf of bread, most people reach for", "continuations": [ " a paintbrush.", " a blanket.", " a shoe.", " a knife." ], "tags": [ "tool_use" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -14.79000186920166, "mean_logprob": -3.697500467300415, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 4, "sum_logprob": -14.01634693145752, "mean_logprob": -3.50408673286438, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 4, "sum_logprob": -10.878457069396973, "mean_logprob": -2.719614267349243, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 4, "sum_logprob": -2.3763246536254883, "mean_logprob": -0.5940811634063721, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 2.125533103942871, "context_truncated": false, "elapsed_seconds": 0.031615363999662804 }, { "id": "commonsense-005", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "If a houseplant's soil is very dry, the plant probably needs", "continuations": [ " water.", " a password.", " louder music.", " a winter coat." ], "tags": [ "everyday" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -3.1428794860839844, "mean_logprob": -1.5714397430419922, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 3, "sum_logprob": -22.261730194091797, "mean_logprob": -7.420576731363933, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 5, "sum_logprob": -34.00652313232422, "mean_logprob": -6.801304626464844, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 4, "sum_logprob": -15.291974067687988, "mean_logprob": -3.822993516921997, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 2.251553773880005, "context_truncated": false, "elapsed_seconds": 0.06647472599979665 }, { "id": "commonsense-006", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "Omar put leftovers in the refrigerator to keep them", "continuations": [ " invisible.", " cold.", " expensive.", " rectangular." ], "tags": [ "everyday" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -14.542591094970703, "mean_logprob": -7.271295547485352, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 2, "sum_logprob": -3.701167106628418, "mean_logprob": -1.850583553314209, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 2, "sum_logprob": -16.922225952148438, "mean_logprob": -8.461112976074219, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 2, "sum_logprob": -17.653610229492188, "mean_logprob": -8.826805114746094, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 5.420711994171143, "context_truncated": false, "elapsed_seconds": 0.02647093199993833 }, { "id": "commonsense-007", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "Someone standing outside in heavy rain without shelter will probably get", "continuations": [ " taller.", " younger.", " wet.", " magnetic." ], "tags": [ "physical" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -11.499820709228516, "mean_logprob": -3.8332735697428384, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 2, "sum_logprob": -13.593626976013184, "mean_logprob": -6.796813488006592, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 2, "sum_logprob": -5.4803338050842285, "mean_logprob": -2.7401669025421143, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 2, "sum_logprob": -16.12941551208496, "mean_logprob": -8.06470775604248, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 1.0931066672007241, "context_truncated": false, "elapsed_seconds": 0.028404538999893703 }, { "id": "commonsense-008", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "A cyclist approaching a red traffic light should", "continuations": [ " accelerate through every lane.", " close both eyes.", " remove the wheels.", " stop." ], "tags": [ "safety" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -21.939393997192383, "mean_logprob": -3.1341991424560547, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 4, "sum_logprob": -18.764877319335938, "mean_logprob": -4.691219329833984, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 4, "sum_logprob": -22.600740432739258, "mean_logprob": -5.6501851081848145, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 2, "sum_logprob": -4.850698471069336, "mean_logprob": -2.425349235534668, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 0.7088499069213867, "context_truncated": false, "elapsed_seconds": 0.03384869600085949 }, { "id": "commonsense-009", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "To find out whether someone has a fever, a nurse commonly uses", "continuations": [ " a thermometer.", " a ruler.", " a compass.", " a stapler." ], "tags": [ "tool_use" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -7.429974555969238, "mean_logprob": -1.8574936389923096, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 3, "sum_logprob": -12.03149700164795, "mean_logprob": -4.010499000549316, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 3, "sum_logprob": -12.593362808227539, "mean_logprob": -4.197787602742513, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 5, "sum_logprob": -16.09111976623535, "mean_logprob": -3.2182239532470702, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 1.3607303142547607, "context_truncated": false, "elapsed_seconds": 0.030042739999771584 }, { "id": "commonsense-010", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 0.0, "item_elo": 800, "context": "When the phone battery reaches zero, the phone usually", "continuations": [ " doubles in size.", " turns off.", " prints a book.", " becomes waterproof." ], "tags": [ "everyday" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -10.272951126098633, "mean_logprob": -2.0545902252197266, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 3, "sum_logprob": -6.171103000640869, "mean_logprob": -2.0570343335469565, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 4, "sum_logprob": -26.03506851196289, "mean_logprob": -6.508767127990723, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 4, "sum_logprob": -16.60114288330078, "mean_logprob": -4.150285720825195, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 0.0024441083272299657, "context_truncated": false, "elapsed_seconds": 0.03187223499844549 }, { "id": "commonsense-011", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "A person who has not eaten all day is likely to feel", "continuations": [ " polished.", " transparent.", " hungry.", " metallic." ], "tags": [ "human_needs" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -21.305147171020508, "mean_logprob": -7.101715723673503, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 2, "sum_logprob": -21.010116577148438, "mean_logprob": -10.505058288574219, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 3, "sum_logprob": -5.818660736083984, "mean_logprob": -1.9395535786946614, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 3, "sum_logprob": -15.83749008178711, "mean_logprob": -5.279163360595703, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 3.339609781901042, "context_truncated": false, "elapsed_seconds": 0.029702271000132896 }, { "id": "commonsense-012", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "After washing wet clothes, a common next step is to", "continuations": [ " bury them in concrete.", " mail them overseas.", " paint them black.", " dry them." ], "tags": [ "everyday" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -20.121253967285156, "mean_logprob": -3.3535423278808594, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 5, "sum_logprob": -22.138946533203125, "mean_logprob": -4.427789306640625, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 4, "sum_logprob": -13.949477195739746, "mean_logprob": -3.4873692989349365, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 3, "sum_logprob": -4.874208450317383, "mean_logprob": -1.6247361501057942, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 1.7288061777750652, "context_truncated": false, "elapsed_seconds": 0.0797503419998975 }, { "id": "commonsense-013", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "If a candle flame is blown out, the room may become", "continuations": [ " darker.", " heavier than a car.", " filled with snow.", " twice as wide." ], "tags": [ "causality" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -6.059157371520996, "mean_logprob": -2.019719123840332, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 6, "sum_logprob": -27.804567337036133, "mean_logprob": -4.6340945561726885, "total_sequence_tokens": 21 }, { "index": 2, "tokens": 4, "sum_logprob": -13.497331619262695, "mean_logprob": -3.374332904815674, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 4, "sum_logprob": -22.085975646972656, "mean_logprob": -5.521493911743164, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 1.3546137809753418, "context_truncated": false, "elapsed_seconds": 0.035416651000559796 }, { "id": "commonsense-014", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "To unlock a typical front door from outside, a resident may use", "continuations": [ " soap.", " a key.", " a pillow.", " a forkful of rice." ], "tags": [ "tool_use" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -15.688546180725098, "mean_logprob": -7.844273090362549, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 3, "sum_logprob": -7.20548152923584, "mean_logprob": -2.4018271764119468, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 4, "sum_logprob": -11.886775016784668, "mean_logprob": -2.971693754196167, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 6, "sum_logprob": -26.5023193359375, "mean_logprob": -4.41705322265625, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.5698665777842202, "context_truncated": false, "elapsed_seconds": 0.03503433900004893 }, { "id": "commonsense-015", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "Fresh snow on a sidewalk can make the surface", "continuations": [ " spicy.", " musical.", " slippery.", " edible." ], "tags": [ "physical" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -16.030292510986328, "mean_logprob": -5.343430836995442, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 2, "sum_logprob": -18.539226531982422, "mean_logprob": -9.269613265991211, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 4, "sum_logprob": -3.111222743988037, "mean_logprob": -0.7778056859970093, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 3, "sum_logprob": -11.036014556884766, "mean_logprob": -3.6786715189615884, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 2.900865832964579, "context_truncated": false, "elapsed_seconds": 0.03836592500010738 }, { "id": "commonsense-016", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "If a smoke alarm sounds and smoke is visible, people should", "continuations": [ " finish a long nap.", " hide the alarm.", " open a novel.", " leave the building safely." ], "tags": [ "safety" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -31.506465911865234, "mean_logprob": -5.251077651977539, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 4, "sum_logprob": -15.628789901733398, "mean_logprob": -3.9071974754333496, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 4, "sum_logprob": -35.51291275024414, "mean_logprob": -8.878228187561035, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 5, "sum_logprob": -13.135072708129883, "mean_logprob": -2.6270145416259765, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 1.2801829338073731, "context_truncated": false, "elapsed_seconds": 0.028125348999310518 }, { "id": "commonsense-017", "category": "commonsense", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "A ceramic mug filled with hot tea may be too hot to", "continuations": [ " hold comfortably.", " remember a name.", " become ceramic.", " count backward." ], "tags": [ "physical" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -13.978292465209961, "mean_logprob": -3.4945731163024902, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 4, "sum_logprob": -26.312808990478516, "mean_logprob": -6.578202247619629, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 4, "sum_logprob": -22.872270584106445, "mean_logprob": -5.718067646026611, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 3, "sum_logprob": -23.461456298828125, "mean_logprob": -7.820485432942708, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 2.223494529724121, "context_truncated": false, "elapsed_seconds": 0.03861710899946047 }, { "id": "commonsense-018", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "Tara heard thunder only a few seconds after seeing lightning, suggesting the storm was", "continuations": [ " beneath the ocean.", " nearby.", " made of glass.", " over last year." ], "tags": [ "causality" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -12.610671043395996, "mean_logprob": -3.152667760848999, "total_sequence_tokens": 22 }, { "index": 1, "tokens": 2, "sum_logprob": -7.430370330810547, "mean_logprob": -3.7151851654052734, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 4, "sum_logprob": -14.351226806640625, "mean_logprob": -3.5878067016601562, "total_sequence_tokens": 22 }, { "index": 3, "tokens": 4, "sum_logprob": -21.74671745300293, "mean_logprob": -5.436679363250732, "total_sequence_tokens": 22 } ], "best_vs_second_margin": 0.4351389408111572, "context_truncated": false, "elapsed_seconds": 0.02832317799948214 }, { "id": "commonsense-019", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "The freezer door was left open overnight. In the morning, the ice cream was probably", "continuations": [ " sharper.", " newly packaged.", " soft or melted.", " louder." ], "tags": [ "causality" ], "prediction": 3, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -15.146666526794434, "mean_logprob": -3.7866666316986084, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 3, "sum_logprob": -21.309471130371094, "mean_logprob": -7.103157043457031, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 5, "sum_logprob": -16.809839248657227, "mean_logprob": -3.361967849731445, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 4, "sum_logprob": -11.778133392333984, "mean_logprob": -2.944533348083496, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.41743450164794904, "context_truncated": false, "elapsed_seconds": 0.035673541999130975 }, { "id": "commonsense-020", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "Ravi arrived early for an interview but did not want to interrupt the meeting inside, so he", "continuations": [ " shouted through the door.", " canceled every meeting.", " moved the building.", " waited quietly." ], "tags": [ "social" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -18.4698486328125, "mean_logprob": -2.6385498046875, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 6, "sum_logprob": -21.761749267578125, "mean_logprob": -3.626958211263021, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 4, "sum_logprob": -17.902360916137695, "mean_logprob": -4.475590229034424, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 5, "sum_logprob": -10.329994201660156, "mean_logprob": -2.065998840332031, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.5725509643554689, "context_truncated": false, "elapsed_seconds": 0.035349686000699876 }, { "id": "commonsense-021", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "A recipe serves four people, but eight guests are expected. A sensible cook would", "continuations": [ " double the ingredients.", " halve every plate.", " freeze the oven.", " omit all food." ], "tags": [ "planning" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -11.00696086883545, "mean_logprob": -2.7517402172088623, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 5, "sum_logprob": -22.688758850097656, "mean_logprob": -4.537751770019531, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 4, "sum_logprob": -24.844209671020508, "mean_logprob": -6.211052417755127, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 5, "sum_logprob": -22.738433837890625, "mean_logprob": -4.547686767578125, "total_sequence_tokens": 22 } ], "best_vs_second_margin": 1.7860115528106686, "context_truncated": false, "elapsed_seconds": 0.027442759999757982 }, { "id": "commonsense-022", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "Mina's laptop became very hot and its fan ran constantly, so she first checked whether", "continuations": [ " the keyboard knew her name.", " its vents were blocked.", " the screen was square.", " noon had arrived." ], "tags": [ "troubleshooting" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -23.518367767333984, "mean_logprob": -3.9197279612223306, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 6, "sum_logprob": -16.88454246520996, "mean_logprob": -2.8140904108683267, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 5, "sum_logprob": -22.184528350830078, "mean_logprob": -4.4369056701660154, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 5, "sum_logprob": -25.38568115234375, "mean_logprob": -5.07713623046875, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 1.105637550354004, "context_truncated": false, "elapsed_seconds": 0.027006951999283046 }, { "id": "commonsense-023", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "A friend says, \"I just lost my job.\" The most considerate immediate response is to", "continuations": [ " ask for a loan.", " change the subject to sports.", " express sympathy and listen.", " congratulate them." ], "tags": [ "social" ], "prediction": 3, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -12.615581512451172, "mean_logprob": -2.5231163024902346, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 6, "sum_logprob": -27.88450813293457, "mean_logprob": -4.647418022155762, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 6, "sum_logprob": -18.08245277404785, "mean_logprob": -3.013742129007975, "total_sequence_tokens": 26 }, { "index": 3, "tokens": 5, "sum_logprob": -11.871034622192383, "mean_logprob": -2.3742069244384765, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.14890937805175808, "context_truncated": false, "elapsed_seconds": 0.03359649000049103 }, { "id": "commonsense-024", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "The label says a medicine may cause drowsiness. After taking it, a person should avoid", "continuations": [ " drinking water.", " reading the label.", " sitting safely.", " driving a car." ], "tags": [ "safety" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -8.047860145568848, "mean_logprob": -2.682620048522949, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 4, "sum_logprob": -15.666956901550293, "mean_logprob": -3.9167392253875732, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 3, "sum_logprob": -23.298513412475586, "mean_logprob": -7.766171137491862, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 4, "sum_logprob": -14.242864608764648, "mean_logprob": -3.560716152191162, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 0.8780961036682129, "context_truncated": false, "elapsed_seconds": 0.02827333200002613 }, { "id": "commonsense-025", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "Water is dripping from the ceiling directly below an upstairs bathroom. The leak likely", "continuations": [ " originates near that bathroom.", " comes from sunlight.", " proves the roof is dry.", " makes water weightless." ], "tags": [ "diagnosis" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -21.697128295898438, "mean_logprob": -3.0995897565569197, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 4, "sum_logprob": -16.447704315185547, "mean_logprob": -4.111926078796387, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 6, "sum_logprob": -27.732826232910156, "mean_logprob": -4.622137705485026, "total_sequence_tokens": 26 }, { "index": 3, "tokens": 5, "sum_logprob": -34.71245574951172, "mean_logprob": -6.942491149902343, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 1.012336322239467, "context_truncated": false, "elapsed_seconds": 0.02621591200113471 }, { "id": "commonsense-026", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "The car's fuel gauge is nearly empty and the next town is far away. The prudent action is to", "continuations": [ " increase tire size.", " refuel before leaving.", " turn off the steering wheel.", " ignore all signs." ], "tags": [ "planning" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -23.950927734375, "mean_logprob": -5.98773193359375, "total_sequence_tokens": 28 }, { "index": 1, "tokens": 5, "sum_logprob": -13.28018856048584, "mean_logprob": -2.656037712097168, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 7, "sum_logprob": -17.440576553344727, "mean_logprob": -2.4915109361921037, "total_sequence_tokens": 31 }, { "index": 3, "tokens": 4, "sum_logprob": -15.294353485107422, "mean_logprob": -3.8235883712768555, "total_sequence_tokens": 28 } ], "best_vs_second_margin": 0.16452677590506415, "context_truncated": false, "elapsed_seconds": 0.03307416600000579 }, { "id": "commonsense-027", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "A package marked \"fragile\" should generally be", "continuations": [ " shaken to test it.", " left in rain.", " handled carefully.", " used as a chair." ], "tags": [ "everyday" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -24.553762435913086, "mean_logprob": -4.092293739318848, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 4, "sum_logprob": -21.365097045898438, "mean_logprob": -5.341274261474609, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 3, "sum_logprob": -10.0580472946167, "mean_logprob": -3.3526824315389, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 5, "sum_logprob": -23.24994468688965, "mean_logprob": -4.6499889373779295, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 0.7396113077799478, "context_truncated": false, "elapsed_seconds": 0.027997896000670153 }, { "id": "commonsense-028", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "Kai cannot hear the television, but everyone else can. Before replacing it, Kai should check", "continuations": [ " whether the walls are blue.", " how much the sofa weighs.", " if tomorrow is Tuesday.", " the volume and audio settings." ], "tags": [ "troubleshooting" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -26.885547637939453, "mean_logprob": -4.480924606323242, "total_sequence_tokens": 26 }, { "index": 1, "tokens": 7, "sum_logprob": -25.82662582397461, "mean_logprob": -3.6895179748535156, "total_sequence_tokens": 27 }, { "index": 2, "tokens": 5, "sum_logprob": -20.93070411682129, "mean_logprob": -4.186140823364258, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 6, "sum_logprob": -16.532907485961914, "mean_logprob": -2.7554845809936523, "total_sequence_tokens": 26 } ], "best_vs_second_margin": 0.9340333938598633, "context_truncated": false, "elapsed_seconds": 0.03174066499923356 }, { "id": "commonsense-029", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "A wet floor sign in a hallway warns pedestrians that they should", "continuations": [ " walk carefully.", " run faster.", " turn off every light.", " remove their shoes immediately." ], "tags": [ "safety" ], "prediction": 3, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -11.348014831542969, "mean_logprob": -3.7826716105143228, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 3, "sum_logprob": -11.512643814086914, "mean_logprob": -3.837547938028971, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 5, "sum_logprob": -27.827768325805664, "mean_logprob": -5.565553665161133, "total_sequence_tokens": 22 }, { "index": 3, "tokens": 5, "sum_logprob": -9.475430488586426, "mean_logprob": -1.8950860977172852, "total_sequence_tokens": 22 } ], "best_vs_second_margin": 1.8875855127970376, "context_truncated": false, "elapsed_seconds": 0.02886738199958927 }, { "id": "commonsense-030", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "Nico promised to return a borrowed book on Friday but needs it until Monday. The responsible choice is to", "continuations": [ " hide the book.", " ask the owner for an extension.", " claim Friday was canceled.", " lend it to a stranger." ], "tags": [ "social" ], "prediction": 3, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -13.196928024291992, "mean_logprob": -3.299232006072998, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 7, "sum_logprob": -20.57825469970703, "mean_logprob": -2.9397506713867188, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 7, "sum_logprob": -32.479103088378906, "mean_logprob": -4.639871869768415, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 8, "sum_logprob": -19.80704116821289, "mean_logprob": -2.4758801460266113, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.4638705253601074, "context_truncated": false, "elapsed_seconds": 0.033459694001066964 }, { "id": "commonsense-031", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "A refrigerator is running, but food inside is warm. One useful first check is whether", "continuations": [ " the food can read.", " the kitchen is rectangular.", " the door seals completely.", " the shelves are named." ], "tags": [ "troubleshooting" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -22.93718147277832, "mean_logprob": -4.587436294555664, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 5, "sum_logprob": -19.133255004882812, "mean_logprob": -3.8266510009765624, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 6, "sum_logprob": -16.214479446411133, "mean_logprob": -2.702413241068522, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 6, "sum_logprob": -28.418724060058594, "mean_logprob": -4.736454010009766, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 1.1242377599080404, "context_truncated": false, "elapsed_seconds": 0.02941785599978175 }, { "id": "commonsense-032", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "A hiker sees dark clouds approaching and has several hours left on an exposed ridge. The safer choice is to", "continuations": [ " climb the tallest tree.", " hold a metal pole.", " wait for lightning.", " descend toward shelter." ], "tags": [ "safety" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -18.073514938354492, "mean_logprob": -3.0122524897257485, "total_sequence_tokens": 30 }, { "index": 1, "tokens": 5, "sum_logprob": -25.604442596435547, "mean_logprob": -5.120888519287109, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 5, "sum_logprob": -17.805259704589844, "mean_logprob": -3.561051940917969, "total_sequence_tokens": 29 }, { "index": 3, "tokens": 4, "sum_logprob": -18.156234741210938, "mean_logprob": -4.539058685302734, "total_sequence_tokens": 28 } ], "best_vs_second_margin": 0.5487994511922203, "context_truncated": false, "elapsed_seconds": 0.03707048300020688 }, { "id": "commonsense-033", "category": "commonsense", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "An egg dropped onto a thick cushion is less likely to break because the cushion", "continuations": [ " reduces the sudden impact.", " makes the egg colder.", " removes gravity.", " turns shell into rubber." ], "tags": [ "physical" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -19.05860710144043, "mean_logprob": -3.811721420288086, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 6, "sum_logprob": -20.780445098876953, "mean_logprob": -3.463407516479492, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 3, "sum_logprob": -20.995624542236328, "mean_logprob": -6.998541514078776, "total_sequence_tokens": 22 }, { "index": 3, "tokens": 5, "sum_logprob": -35.17015838623047, "mean_logprob": -7.034031677246094, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 0.34831390380859384, "context_truncated": false, "elapsed_seconds": 0.047453494000365026 }, { "id": "commonsense-034", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "A metal jar lid is stuck. Running warm water over the lid can help because the metal may", "continuations": [ " lose all mass.", " expand slightly.", " stop being metal.", " become colder than ice." ], "tags": [ "physical" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -26.746746063232422, "mean_logprob": -6.6866865158081055, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 3, "sum_logprob": -7.445197105407715, "mean_logprob": -2.4817323684692383, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 4, "sum_logprob": -22.945758819580078, "mean_logprob": -5.7364397048950195, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 6, "sum_logprob": -18.385082244873047, "mean_logprob": -3.064180374145508, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.5824480056762695, "context_truncated": false, "elapsed_seconds": 0.027411420000134967 }, { "id": "commonsense-035", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "Lena receives an email demanding immediate payment through gift cards and threatening arrest. The message is most likely", "continuations": [ " a routine tax receipt.", " a weather alert.", " a scam.", " a library reminder." ], "tags": [ "digital_safety" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -27.713577270507812, "mean_logprob": -4.618929545084636, "total_sequence_tokens": 26 }, { "index": 1, "tokens": 4, "sum_logprob": -17.01344108581543, "mean_logprob": -4.253360271453857, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 4, "sum_logprob": -7.814584732055664, "mean_logprob": -1.953646183013916, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 4, "sum_logprob": -25.283443450927734, "mean_logprob": -6.320860862731934, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 2.2997140884399414, "context_truncated": false, "elapsed_seconds": 0.03003643500051112 }, { "id": "commonsense-036", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "A sealed plastic bottle dents inward after hot air inside cools. The best explanation is that", "continuations": [ " the bottle gained extra air.", " cooling erased the plastic.", " gravity reversed briefly.", " pressure inside decreased." ], "tags": [ "physical" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -26.828231811523438, "mean_logprob": -4.471371968587239, "total_sequence_tokens": 26 }, { "index": 1, "tokens": 6, "sum_logprob": -29.029586791992188, "mean_logprob": -4.838264465332031, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 4, "sum_logprob": -37.96666717529297, "mean_logprob": -9.491666793823242, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 4, "sum_logprob": -23.304729461669922, "mean_logprob": -5.8261823654174805, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 0.36689249674479196, "context_truncated": false, "elapsed_seconds": 0.027758326999901328 }, { "id": "commonsense-037", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "A colleague repeatedly misses deadlines because requirements change without notice. The most constructive first step is to", "continuations": [ " document changes and clarify priorities.", " publicly insult the colleague.", " silently redo all work forever.", " delete the project." ], "tags": [ "workplace" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -22.14117431640625, "mean_logprob": -3.6901957194010415, "total_sequence_tokens": 28 }, { "index": 1, "tokens": 7, "sum_logprob": -24.30960464477539, "mean_logprob": -3.4728006635393416, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 8, "sum_logprob": -52.79871368408203, "mean_logprob": -6.599839210510254, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 4, "sum_logprob": -17.443693161010742, "mean_logprob": -4.3609232902526855, "total_sequence_tokens": 26 } ], "best_vs_second_margin": 0.21739505586169994, "context_truncated": false, "elapsed_seconds": 0.028819236000344972 }, { "id": "commonsense-038", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "A carbon monoxide alarm sounds, but nobody can smell smoke. Occupants should still", "continuations": [ " assume silence means safety.", " leave and contact emergency services.", " search with a flame.", " unplug it and sleep." ], "tags": [ "safety" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -34.20486831665039, "mean_logprob": -6.840973663330078, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 6, "sum_logprob": -21.1650333404541, "mean_logprob": -3.52750555674235, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 6, "sum_logprob": -27.941219329833984, "mean_logprob": -4.656869888305664, "total_sequence_tokens": 26 }, { "index": 3, "tokens": 7, "sum_logprob": -27.197368621826172, "mean_logprob": -3.885338374546596, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.3578328178042458, "context_truncated": false, "elapsed_seconds": 0.02605757099991024 }, { "id": "commonsense-039", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "A wooden drawer sticks only during humid weather. The most plausible reason is that the wood", "continuations": [ " becomes electrically charged.", " forgets its shape.", " swells after absorbing moisture.", " turns into metal." ], "tags": [ "physical" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -19.181184768676758, "mean_logprob": -3.196864128112793, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 5, "sum_logprob": -23.117597579956055, "mean_logprob": -4.623519515991211, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 7, "sum_logprob": -30.014820098876953, "mean_logprob": -4.2878314426967075, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 4, "sum_logprob": -20.909446716308594, "mean_logprob": -5.227361679077148, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 1.0909673145839145, "context_truncated": false, "elapsed_seconds": 0.03314090099956957 }, { "id": "commonsense-040", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "Sam hears a rumor about a coworker from someone who admits they were not present. Before repeating it, Sam should", "continuations": [ " add dramatic details.", " post it anonymously.", " treat it as confirmed.", " verify the information." ], "tags": [ "social" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -29.395584106445312, "mean_logprob": -7.348896026611328, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 6, "sum_logprob": -21.053836822509766, "mean_logprob": -3.5089728037516275, "total_sequence_tokens": 33 }, { "index": 2, "tokens": 5, "sum_logprob": -22.396867752075195, "mean_logprob": -4.479373550415039, "total_sequence_tokens": 32 }, { "index": 3, "tokens": 4, "sum_logprob": -10.483184814453125, "mean_logprob": -2.6207962036132812, "total_sequence_tokens": 31 } ], "best_vs_second_margin": 0.8881766001383462, "context_truncated": false, "elapsed_seconds": 0.04679363399918657 }, { "id": "commonsense-041", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "Two identical wet towels are hung up, one spread flat and one crumpled. The spread towel usually dries first because it has", "continuations": [ " more surface exposed to air.", " less fabric.", " no water initially.", " a different mass forever." ], "tags": [ "physical" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -14.11756706237793, "mean_logprob": -2.352927843729655, "total_sequence_tokens": 36 }, { "index": 1, "tokens": 3, "sum_logprob": -10.636102676391602, "mean_logprob": -3.5453675587972007, "total_sequence_tokens": 33 }, { "index": 2, "tokens": 4, "sum_logprob": -17.9916934967041, "mean_logprob": -4.497923374176025, "total_sequence_tokens": 34 }, { "index": 3, "tokens": 5, "sum_logprob": -29.345022201538086, "mean_logprob": -5.869004440307617, "total_sequence_tokens": 35 } ], "best_vs_second_margin": 1.1924397150675459, "context_truncated": false, "elapsed_seconds": 0.04707452600086981 }, { "id": "commonsense-042", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "A meeting invite lists 3 p.m. in another time zone. To avoid arriving at the wrong time, a participant should", "continuations": [ " ignore the time zone.", " convert it to local time.", " attend at every hour.", " change the calendar date randomly." ], "tags": [ "planning" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -17.2685604095459, "mean_logprob": -3.4537120819091798, "total_sequence_tokens": 32 }, { "index": 1, "tokens": 6, "sum_logprob": -22.19762420654297, "mean_logprob": -3.699604034423828, "total_sequence_tokens": 33 }, { "index": 2, "tokens": 5, "sum_logprob": -27.377838134765625, "mean_logprob": -5.475567626953125, "total_sequence_tokens": 32 }, { "index": 3, "tokens": 6, "sum_logprob": -31.6478328704834, "mean_logprob": -5.274638811747233, "total_sequence_tokens": 33 } ], "best_vs_second_margin": 0.24589195251464835, "context_truncated": false, "elapsed_seconds": 0.04162225400068564 }, { "id": "commonsense-043", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "A pan of oil catches fire on a stove. The cook should turn off the heat and", "continuations": [ " pour water into the oil.", " carry the burning pan outside.", " cover the pan with a lid.", " fan the flames." ], "tags": [ "safety" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -14.20297622680664, "mean_logprob": -2.3671627044677734, "total_sequence_tokens": 26 }, { "index": 1, "tokens": 6, "sum_logprob": -21.66763687133789, "mean_logprob": -3.6112728118896484, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 7, "sum_logprob": -10.082666397094727, "mean_logprob": -1.4403809138706751, "total_sequence_tokens": 27 }, { "index": 3, "tokens": 5, "sum_logprob": -12.846022605895996, "mean_logprob": -2.5692045211791994, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.9267817905970983, "context_truncated": false, "elapsed_seconds": 0.028579490000993246 }, { "id": "commonsense-044", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "After a software update, a printer stops appearing on one computer but works from others. The issue is most likely with", "continuations": [ " all paper ever made.", " the building's plumbing.", " the printer's physical existence.", " that computer's configuration." ], "tags": [ "troubleshooting" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -44.12640380859375, "mean_logprob": -8.82528076171875, "total_sequence_tokens": 28 }, { "index": 1, "tokens": 8, "sum_logprob": -24.53286361694336, "mean_logprob": -3.06660795211792, "total_sequence_tokens": 31 }, { "index": 2, "tokens": 7, "sum_logprob": -23.706457138061523, "mean_logprob": -3.386636734008789, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 6, "sum_logprob": -15.543107986450195, "mean_logprob": -2.590517997741699, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.4760899543762207, "context_truncated": false, "elapsed_seconds": 0.033685976999549894 }, { "id": "commonsense-045", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "A person feels dizzy after standing up very quickly. Sitting back down helps primarily by", "continuations": [ " reducing the immediate risk of falling.", " making the floor warmer.", " stopping all circulation.", " increasing their height." ], "tags": [ "safety" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -16.934162139892578, "mean_logprob": -2.419166019984654, "total_sequence_tokens": 26 }, { "index": 1, "tokens": 5, "sum_logprob": -19.554096221923828, "mean_logprob": -3.910819244384766, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 4, "sum_logprob": -20.506786346435547, "mean_logprob": -5.126696586608887, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 4, "sum_logprob": -17.24651527404785, "mean_logprob": -4.311628818511963, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 1.4916532244001117, "context_truncated": false, "elapsed_seconds": 0.03306646099917998 }, { "id": "commonsense-046", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "A sealed bag of chips puffs up during a drive to high altitude because outside air pressure", "continuations": [ " becomes much higher.", " becomes lower.", " changes the chips into gas.", " disappears completely." ], "tags": [ "physical" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -13.810972213745117, "mean_logprob": -3.4527430534362793, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 3, "sum_logprob": -9.870441436767578, "mean_logprob": -3.290147145589193, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 6, "sum_logprob": -30.123023986816406, "mean_logprob": -5.020503997802734, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 4, "sum_logprob": -16.583572387695312, "mean_logprob": -4.145893096923828, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.16259590784708644, "context_truncated": false, "elapsed_seconds": 0.02561516199966718 }, { "id": "commonsense-047", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "A manager asks for feedback but becomes defensive whenever criticism is offered. Employees will likely become", "continuations": [ " more candid each time.", " unable to remember work.", " reluctant to speak honestly.", " certain all criticism vanished." ], "tags": [ "social" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -25.41448974609375, "mean_logprob": -5.08289794921875, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 5, "sum_logprob": -31.7397403717041, "mean_logprob": -6.34794807434082, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 8, "sum_logprob": -15.93793773651123, "mean_logprob": -1.9922422170639038, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 6, "sum_logprob": -42.678192138671875, "mean_logprob": -7.1130320231119795, "total_sequence_tokens": 26 } ], "best_vs_second_margin": 3.090655732154846, "context_truncated": false, "elapsed_seconds": 0.027545950999410707 }, { "id": "commonsense-048", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "A parked car has one window fogged on the inside while the others are clear. This suggests moisture is concentrated", "continuations": [ " inside the engine only.", " on the road beneath.", " in the sunlight.", " near that window." ], "tags": [ "diagnosis" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -18.106586456298828, "mean_logprob": -3.6213172912597655, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 5, "sum_logprob": -16.36924934387207, "mean_logprob": -3.273849868774414, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 4, "sum_logprob": -12.548701286315918, "mean_logprob": -3.1371753215789795, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 4, "sum_logprob": -12.029919624328613, "mean_logprob": -3.0074799060821533, "total_sequence_tokens": 28 } ], "best_vs_second_margin": 0.12969541549682617, "context_truncated": false, "elapsed_seconds": 0.034464940001271316 }, { "id": "commonsense-049", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "A child apologizes only after being caught and immediately repeats the behavior. The apology alone is weak evidence of", "continuations": [ " genuine change.", " having been caught.", " the behavior occurring.", " spoken language." ], "tags": [ "social" ], "prediction": 2, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -12.416661262512207, "mean_logprob": -4.138887087504069, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 4, "sum_logprob": -17.955171585083008, "mean_logprob": -4.488792896270752, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 4, "sum_logprob": -14.714911460876465, "mean_logprob": -3.678727865219116, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 3, "sum_logprob": -21.599285125732422, "mean_logprob": -7.199761708577474, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.4601592222849531, "context_truncated": false, "elapsed_seconds": 0.03128207199915778 }, { "id": "commonsense-050", "category": "commonsense", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "A rechargeable battery lasts much less time in freezing weather but recovers indoors. The temporary loss is most plausibly due to", "continuations": [ " permanent deletion of electricity.", " slower chemical reactions in the cold.", " the device becoming heavier.", " indoor clocks running faster." ], "tags": [ "physical" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -31.399362564086914, "mean_logprob": -5.233227094014485, "total_sequence_tokens": 36 }, { "index": 1, "tokens": 7, "sum_logprob": -24.70549201965332, "mean_logprob": -3.529356002807617, "total_sequence_tokens": 37 }, { "index": 2, "tokens": 6, "sum_logprob": -23.229263305664062, "mean_logprob": -3.8715438842773438, "total_sequence_tokens": 36 }, { "index": 3, "tokens": 6, "sum_logprob": -34.779884338378906, "mean_logprob": -5.796647389729817, "total_sequence_tokens": 36 } ], "best_vs_second_margin": 0.34218788146972656, "context_truncated": false, "elapsed_seconds": 0.04696313400017971 }, { "id": "world_knowledge-001", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "The capital city of France is", "continuations": [ " Vienna.", " Madrid.", " Paris.", " Lisbon." ], "tags": [ "geography" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -11.509716033935547, "mean_logprob": -3.836572011311849, "total_sequence_tokens": 9 }, { "index": 1, "tokens": 3, "sum_logprob": -7.032500743865967, "mean_logprob": -2.344166914621989, "total_sequence_tokens": 9 }, { "index": 2, "tokens": 2, "sum_logprob": -3.3560290336608887, "mean_logprob": -1.6780145168304443, "total_sequence_tokens": 8 }, { "index": 3, "tokens": 4, "sum_logprob": -9.288416862487793, "mean_logprob": -2.3221042156219482, "total_sequence_tokens": 10 } ], "best_vs_second_margin": 0.6440896987915039, "context_truncated": false, "elapsed_seconds": 0.03533909900033905 }, { "id": "world_knowledge-002", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 0.0, "item_elo": 800, "context": "The planet known for its prominent rings is", "continuations": [ " Mercury.", " Saturn.", " Earth.", " Mars." ], "tags": [ "science" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -7.320735454559326, "mean_logprob": -2.4402451515197754, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 2, "sum_logprob": -8.424192428588867, "mean_logprob": -4.212096214294434, "total_sequence_tokens": 10 }, { "index": 2, "tokens": 2, "sum_logprob": -6.013822555541992, "mean_logprob": -3.006911277770996, "total_sequence_tokens": 10 }, { "index": 3, "tokens": 2, "sum_logprob": -8.402318954467773, "mean_logprob": -4.201159477233887, "total_sequence_tokens": 10 } ], "best_vs_second_margin": 0.5666661262512207, "context_truncated": false, "elapsed_seconds": 0.027122282999698655 }, { "id": "world_knowledge-003", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "Water freezes at zero degrees on the", "continuations": [ " Kelvin scale.", " Fahrenheit scale.", " Celsius scale.", " Beaufort scale." ], "tags": [ "science" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -8.779051780700684, "mean_logprob": -1.7558103561401368, "total_sequence_tokens": 13 }, { "index": 1, "tokens": 7, "sum_logprob": -10.190202713012695, "mean_logprob": -1.4557432447160994, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 5, "sum_logprob": -6.731854438781738, "mean_logprob": -1.3463708877563476, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 6, "sum_logprob": -15.025651931762695, "mean_logprob": -2.504275321960449, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 0.10937235695975178, "context_truncated": false, "elapsed_seconds": 0.02718069999900763 }, { "id": "world_knowledge-004", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 0.0, "item_elo": 800, "context": "The author of the play Hamlet was", "continuations": [ " Charles Dickens.", " Jane Austen.", " Mark Twain.", " William Shakespeare." ], "tags": [ "literature" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -11.206514358520508, "mean_logprob": -2.801628589630127, "total_sequence_tokens": 12 }, { "index": 1, "tokens": 4, "sum_logprob": -12.495871543884277, "mean_logprob": -3.1239678859710693, "total_sequence_tokens": 12 }, { "index": 2, "tokens": 4, "sum_logprob": -15.107165336608887, "mean_logprob": -3.7767913341522217, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 3, "sum_logprob": -10.643210411071777, "mean_logprob": -3.5477368036905923, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 0.3223392963409424, "context_truncated": false, "elapsed_seconds": 0.026172885000050883 }, { "id": "world_knowledge-005", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "The largest ocean on Earth is the", "continuations": [ " Pacific Ocean.", " Indian Ocean.", " Arctic Ocean.", " Atlantic Ocean." ], "tags": [ "geography" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -4.973231792449951, "mean_logprob": -1.6577439308166504, "total_sequence_tokens": 10 }, { "index": 1, "tokens": 3, "sum_logprob": -5.313286304473877, "mean_logprob": -1.7710954348246257, "total_sequence_tokens": 10 }, { "index": 2, "tokens": 3, "sum_logprob": -5.450287342071533, "mean_logprob": -1.8167624473571777, "total_sequence_tokens": 10 }, { "index": 3, "tokens": 3, "sum_logprob": -5.077324867248535, "mean_logprob": -1.6924416224161785, "total_sequence_tokens": 10 } ], "best_vs_second_margin": 0.03469769159952807, "context_truncated": false, "elapsed_seconds": 0.03793094100001326 }, { "id": "world_knowledge-006", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "Plants primarily absorb water through their", "continuations": [ " flowers.", " roots.", " seeds.", " fruit." ], "tags": [ "biology" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -12.8627290725708, "mean_logprob": -6.4313645362854, "total_sequence_tokens": 9 }, { "index": 1, "tokens": 2, "sum_logprob": -1.4958585500717163, "mean_logprob": -0.7479292750358582, "total_sequence_tokens": 9 }, { "index": 2, "tokens": 2, "sum_logprob": -11.364479064941406, "mean_logprob": -5.682239532470703, "total_sequence_tokens": 9 }, { "index": 3, "tokens": 2, "sum_logprob": -13.464406967163086, "mean_logprob": -6.732203483581543, "total_sequence_tokens": 9 } ], "best_vs_second_margin": 4.934310257434845, "context_truncated": false, "elapsed_seconds": 0.031080873000973952 }, { "id": "world_knowledge-007", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "The chemical symbol for oxygen is", "continuations": [ " Ox.", " G.", " O.", " Og." ], "tags": [ "chemistry" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -10.892958641052246, "mean_logprob": -3.630986213684082, "total_sequence_tokens": 9 }, { "index": 1, "tokens": 2, "sum_logprob": -13.250329971313477, "mean_logprob": -6.625164985656738, "total_sequence_tokens": 8 }, { "index": 2, "tokens": 2, "sum_logprob": -4.206754684448242, "mean_logprob": -2.103377342224121, "total_sequence_tokens": 8 }, { "index": 3, "tokens": 3, "sum_logprob": -11.048662185668945, "mean_logprob": -3.682887395222982, "total_sequence_tokens": 9 } ], "best_vs_second_margin": 1.527608871459961, "context_truncated": false, "elapsed_seconds": 0.027357783999832463 }, { "id": "world_knowledge-008", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "The Nile River flows through the continent of", "continuations": [ " South America.", " Australia.", " Europe.", " Africa." ], "tags": [ "geography" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -3.3431458473205566, "mean_logprob": -1.1143819491068523, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 2, "sum_logprob": -6.791843414306641, "mean_logprob": -3.3959217071533203, "total_sequence_tokens": 10 }, { "index": 2, "tokens": 2, "sum_logprob": -5.591670036315918, "mean_logprob": -2.795835018157959, "total_sequence_tokens": 10 }, { "index": 3, "tokens": 2, "sum_logprob": -1.2561030387878418, "mean_logprob": -0.6280515193939209, "total_sequence_tokens": 10 } ], "best_vs_second_margin": 0.4863304297129314, "context_truncated": false, "elapsed_seconds": 0.0381768369989004 }, { "id": "world_knowledge-009", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "A triangle has", "continuations": [ " three sides.", " four sides.", " five sides.", " six sides." ], "tags": [ "mathematics" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -5.732003211975098, "mean_logprob": -1.9106677373250325, "total_sequence_tokens": 6 }, { "index": 1, "tokens": 3, "sum_logprob": -9.198739051818848, "mean_logprob": -3.0662463506062827, "total_sequence_tokens": 6 }, { "index": 2, "tokens": 3, "sum_logprob": -10.230048179626465, "mean_logprob": -3.4100160598754883, "total_sequence_tokens": 6 }, { "index": 3, "tokens": 3, "sum_logprob": -9.581594467163086, "mean_logprob": -3.1938648223876953, "total_sequence_tokens": 6 } ], "best_vs_second_margin": 1.1555786132812502, "context_truncated": false, "elapsed_seconds": 0.030920102000891347 }, { "id": "world_knowledge-010", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "The organ that pumps blood through the human body is the", "continuations": [ " liver.", " heart.", " lung.", " stomach." ], "tags": [ "biology" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -4.063324928283691, "mean_logprob": -2.0316624641418457, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 2, "sum_logprob": -1.312350869178772, "mean_logprob": -0.656175434589386, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 2, "sum_logprob": -9.319375038146973, "mean_logprob": -4.659687519073486, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 2, "sum_logprob": -7.700719833374023, "mean_logprob": -3.8503599166870117, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 1.3754870295524597, "context_truncated": false, "elapsed_seconds": 0.024139918001310434 }, { "id": "world_knowledge-011", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 0.0, "item_elo": 800, "context": "Japan is an island country in", "continuations": [ " South America.", " western Africa.", " East Asia.", " northern Europe." ], "tags": [ "geography" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -5.345488548278809, "mean_logprob": -1.7818295160929363, "total_sequence_tokens": 9 }, { "index": 1, "tokens": 3, "sum_logprob": -8.188445091247559, "mean_logprob": -2.7294816970825195, "total_sequence_tokens": 9 }, { "index": 2, "tokens": 3, "sum_logprob": -7.010624885559082, "mean_logprob": -2.3368749618530273, "total_sequence_tokens": 9 }, { "index": 3, "tokens": 3, "sum_logprob": -9.137768745422363, "mean_logprob": -3.0459229151407876, "total_sequence_tokens": 9 } ], "best_vs_second_margin": 0.5550454457600911, "context_truncated": false, "elapsed_seconds": 0.05065048700089392 }, { "id": "world_knowledge-012", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "The first month of the Gregorian calendar is", "continuations": [ " March.", " June.", " December.", " January." ], "tags": [ "calendar" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -7.40371036529541, "mean_logprob": -3.701855182647705, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 2, "sum_logprob": -7.013602256774902, "mean_logprob": -3.506801128387451, "total_sequence_tokens": 11 }, { "index": 2, "tokens": 2, "sum_logprob": -7.550854682922363, "mean_logprob": -3.7754273414611816, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 2, "sum_logprob": -5.948097229003906, "mean_logprob": -2.974048614501953, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 0.532752513885498, "context_truncated": false, "elapsed_seconds": 0.071897378000358 }, { "id": "world_knowledge-013", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "Bees commonly produce", "continuations": [ " honey.", " salt.", " wool.", " paper." ], "tags": [ "biology" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -5.9243621826171875, "mean_logprob": -2.9621810913085938, "total_sequence_tokens": 6 }, { "index": 1, "tokens": 2, "sum_logprob": -11.732301712036133, "mean_logprob": -5.866150856018066, "total_sequence_tokens": 6 }, { "index": 2, "tokens": 2, "sum_logprob": -9.110575675964355, "mean_logprob": -4.555287837982178, "total_sequence_tokens": 6 }, { "index": 3, "tokens": 2, "sum_logprob": -11.831554412841797, "mean_logprob": -5.915777206420898, "total_sequence_tokens": 6 } ], "best_vs_second_margin": 1.593106746673584, "context_truncated": false, "elapsed_seconds": 0.03012902299997222 }, { "id": "world_knowledge-014", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "The Great Pyramid of Giza is located in", "continuations": [ " Peru.", " Egypt.", " India.", " Greece." ], "tags": [ "history" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -13.07826042175293, "mean_logprob": -4.35942014058431, "total_sequence_tokens": 13 }, { "index": 1, "tokens": 2, "sum_logprob": -4.243206024169922, "mean_logprob": -2.121603012084961, "total_sequence_tokens": 12 }, { "index": 2, "tokens": 2, "sum_logprob": -13.2263765335083, "mean_logprob": -6.61318826675415, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 2, "sum_logprob": -15.253130912780762, "mean_logprob": -7.626565456390381, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 2.2378171284993487, "context_truncated": false, "elapsed_seconds": 0.03506938899954548 }, { "id": "world_knowledge-015", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 0.0, "item_elo": 800, "context": "The gas humans need to breathe for cellular respiration is", "continuations": [ " helium.", " neon.", " oxygen.", " argon." ], "tags": [ "biology" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -12.004706382751465, "mean_logprob": -3.001176595687866, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 3, "sum_logprob": -22.805034637451172, "mean_logprob": -7.601678212483724, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 2, "sum_logprob": -6.715459823608398, "mean_logprob": -3.357729911804199, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 3, "sum_logprob": -11.974287986755371, "mean_logprob": -3.991429328918457, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 0.356553316116333, "context_truncated": false, "elapsed_seconds": 0.027213841000047978 }, { "id": "world_knowledge-016", "category": "world_knowledge", "difficulty": "easy", "category_weight": 1.1, "difficulty_weight": 1.0, "point_weight": 1.1, "earned_points": 1.1, "item_elo": 800, "context": "A standard chessboard contains", "continuations": [ " forty squares.", " fifty squares.", " one hundred squares.", " sixty-four squares." ], "tags": [ "culture" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -15.206071853637695, "mean_logprob": -5.0686906178792315, "total_sequence_tokens": 8 }, { "index": 1, "tokens": 3, "sum_logprob": -12.570915222167969, "mean_logprob": -4.190305074055989, "total_sequence_tokens": 8 }, { "index": 2, "tokens": 4, "sum_logprob": -15.034111022949219, "mean_logprob": -3.7585277557373047, "total_sequence_tokens": 9 }, { "index": 3, "tokens": 7, "sum_logprob": -17.005380630493164, "mean_logprob": -2.429340090070452, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 1.3291876656668529, "context_truncated": false, "elapsed_seconds": 0.027590434001467656 }, { "id": "world_knowledge-017", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The treaty that formally ended World War I between Germany and the Allied powers was the Treaty of", "continuations": [ " Versailles.", " Utrecht.", " Tordesillas.", " Ghent." ], "tags": [ "history" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -0.7719295024871826, "mean_logprob": -0.15438590049743653, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 4, "sum_logprob": -11.48581600189209, "mean_logprob": -2.8714540004730225, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 6, "sum_logprob": -12.408638000488281, "mean_logprob": -2.0681063334147134, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 3, "sum_logprob": -8.774027824401855, "mean_logprob": -2.924675941467285, "total_sequence_tokens": 22 } ], "best_vs_second_margin": 1.9137204329172768, "context_truncated": false, "elapsed_seconds": 0.027041934999942896 }, { "id": "world_knowledge-018", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The process by which plants convert light energy into chemical energy is", "continuations": [ " fermentation.", " photosynthesis.", " respiration only.", " erosion." ], "tags": [ "biology" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -15.712061882019043, "mean_logprob": -5.237353960673015, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 4, "sum_logprob": -12.029754638671875, "mean_logprob": -3.0074386596679688, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 4, "sum_logprob": -25.57769203186035, "mean_logprob": -6.394423007965088, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 2, "sum_logprob": -20.068880081176758, "mean_logprob": -10.034440040588379, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 2.229915301005046, "context_truncated": false, "elapsed_seconds": 0.031013443000119878 }, { "id": "world_knowledge-019", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "The longest river entirely within Europe is the", "continuations": [ " Danube.", " Rhine.", " Volga.", " Seine." ], "tags": [ "geography" ], "prediction": 1, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -4.150540828704834, "mean_logprob": -1.383513609568278, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 3, "sum_logprob": -2.585611343383789, "mean_logprob": -0.8618704477945963, "total_sequence_tokens": 11 }, { "index": 2, "tokens": 3, "sum_logprob": -5.604860782623291, "mean_logprob": -1.868286927541097, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 3, "sum_logprob": -6.248941898345947, "mean_logprob": -2.0829806327819824, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 0.5216431617736818, "context_truncated": false, "elapsed_seconds": 0.03471019999960845 }, { "id": "world_knowledge-020", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "In computing, RAM is primarily used for", "continuations": [ " permanent archival storage.", " printing documents.", " cooling the processor.", " temporary working memory." ], "tags": [ "computing" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -20.72980308532715, "mean_logprob": -4.1459606170654295, "total_sequence_tokens": 13 }, { "index": 1, "tokens": 3, "sum_logprob": -17.150440216064453, "mean_logprob": -5.716813405354817, "total_sequence_tokens": 11 }, { "index": 2, "tokens": 4, "sum_logprob": -16.436466217041016, "mean_logprob": -4.109116554260254, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 4, "sum_logprob": -15.068862915039062, "mean_logprob": -3.7672157287597656, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 0.3419008255004883, "context_truncated": false, "elapsed_seconds": 0.03145416299958015 }, { "id": "world_knowledge-021", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "The scientist who formulated the three laws of motion was", "continuations": [ " Isaac Newton.", " Louis Pasteur.", " Gregor Mendel.", " Niels Bohr." ], "tags": [ "science_history" ], "prediction": 3, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -10.057504653930664, "mean_logprob": -2.514376163482666, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 5, "sum_logprob": -8.531913757324219, "mean_logprob": -1.7063827514648438, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 6, "sum_logprob": -10.613195419311523, "mean_logprob": -1.7688659032185872, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 6, "sum_logprob": -9.57681655883789, "mean_logprob": -1.5961360931396484, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 0.11024665832519531, "context_truncated": false, "elapsed_seconds": 0.044894495000335155 }, { "id": "world_knowledge-022", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The strait separating Spain from Morocco is the Strait of", "continuations": [ " Malacca.", " Gibraltar.", " Hormuz.", " Bering." ], "tags": [ "geography" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -5.7044291496276855, "mean_logprob": -1.4261072874069214, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 5, "sum_logprob": -1.5994062423706055, "mean_logprob": -0.3198812484741211, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 4, "sum_logprob": -3.2809271812438965, "mean_logprob": -0.8202317953109741, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 3, "sum_logprob": -15.879630088806152, "mean_logprob": -5.293210029602051, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 0.500350546836853, "context_truncated": false, "elapsed_seconds": 0.03594739800064417 }, { "id": "world_knowledge-023", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "A word with the opposite meaning of another word is called an", "continuations": [ " acronym.", " homophone.", " antonym.", " anagram." ], "tags": [ "language" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -7.133785247802734, "mean_logprob": -1.7834463119506836, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 4, "sum_logprob": -7.040304660797119, "mean_logprob": -1.7600761651992798, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 3, "sum_logprob": -0.9590001106262207, "mean_logprob": -0.31966670354207355, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 3, "sum_logprob": -2.9414892196655273, "mean_logprob": -0.9804964065551758, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 0.6608297030131023, "context_truncated": false, "elapsed_seconds": 0.029652658000486554 }, { "id": "world_knowledge-024", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The part of a cell that contains most of its genetic material is the", "continuations": [ " ribosome.", " cell wall.", " vacuole.", " nucleus." ], "tags": [ "biology" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -4.065311431884766, "mean_logprob": -1.0163278579711914, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 3, "sum_logprob": -11.278770446777344, "mean_logprob": -3.7595901489257812, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 4, "sum_logprob": -12.306262969970703, "mean_logprob": -3.076565742492676, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 2, "sum_logprob": -1.0450056791305542, "mean_logprob": -0.5225028395652771, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 0.4938250184059143, "context_truncated": false, "elapsed_seconds": 0.02790486899903044 }, { "id": "world_knowledge-025", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The Renaissance began as a major cultural movement in", "continuations": [ " Italy.", " Iceland.", " Canada.", " Mongolia." ], "tags": [ "history" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -3.302576780319214, "mean_logprob": -1.651288390159607, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 4, "sum_logprob": -12.38290786743164, "mean_logprob": -3.09572696685791, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 2, "sum_logprob": -13.717884063720703, "mean_logprob": -6.858942031860352, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 3, "sum_logprob": -18.62806510925293, "mean_logprob": -6.2093550364176435, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 1.4444385766983032, "context_truncated": false, "elapsed_seconds": 0.028027561998897 }, { "id": "world_knowledge-026", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The SI unit of electrical resistance is the", "continuations": [ " watt.", " ohm.", " tesla.", " joule." ], "tags": [ "physics" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -14.065362930297852, "mean_logprob": -4.688454310099284, "total_sequence_tokens": 11 }, { "index": 1, "tokens": 3, "sum_logprob": -1.9842658042907715, "mean_logprob": -0.6614219347635905, "total_sequence_tokens": 11 }, { "index": 2, "tokens": 4, "sum_logprob": -14.778952598571777, "mean_logprob": -3.6947381496429443, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 4, "sum_logprob": -14.803125381469727, "mean_logprob": -3.7007813453674316, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 3.033316214879354, "context_truncated": false, "elapsed_seconds": 0.028208798999912688 }, { "id": "world_knowledge-027", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The novel Pride and Prejudice was written by", "continuations": [ " George Eliot.", " Virginia Woolf.", " Jane Austen.", " Mary Shelley." ], "tags": [ "literature" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -6.2252607345581055, "mean_logprob": -1.5563151836395264, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 5, "sum_logprob": -4.485273361206055, "mean_logprob": -0.8970546722412109, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 4, "sum_logprob": -2.3515846729278564, "mean_logprob": -0.5878961682319641, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 5, "sum_logprob": -8.91497802734375, "mean_logprob": -1.78299560546875, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 0.3091585040092468, "context_truncated": false, "elapsed_seconds": 0.03339862100074242 }, { "id": "world_knowledge-028", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The layer of Earth's atmosphere containing most weather is the", "continuations": [ " mesosphere.", " thermosphere.", " exosphere.", " troposphere." ], "tags": [ "earth_science" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -5.590150833129883, "mean_logprob": -1.8633836110432942, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 3, "sum_logprob": -6.984744548797607, "mean_logprob": -2.3282481829325357, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 3, "sum_logprob": -7.345436096191406, "mean_logprob": -2.4484786987304688, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 4, "sum_logprob": -2.125917911529541, "mean_logprob": -0.5314794778823853, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 1.331904133160909, "context_truncated": false, "elapsed_seconds": 0.03057092500057479 }, { "id": "world_knowledge-029", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The Magna Carta was sealed in", "continuations": [ " England.", " Brazil.", " China.", " Ethiopia." ], "tags": [ "history" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -11.145002365112305, "mean_logprob": -5.572501182556152, "total_sequence_tokens": 10 }, { "index": 1, "tokens": 2, "sum_logprob": -20.30511474609375, "mean_logprob": -10.152557373046875, "total_sequence_tokens": 10 }, { "index": 2, "tokens": 2, "sum_logprob": -15.851038932800293, "mean_logprob": -7.9255194664001465, "total_sequence_tokens": 10 }, { "index": 3, "tokens": 3, "sum_logprob": -16.8714599609375, "mean_logprob": -5.623819986979167, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 0.05131880442301462, "context_truncated": false, "elapsed_seconds": 0.04208292600014829 }, { "id": "world_knowledge-030", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "In music, a group of three notes played together is commonly called a", "continuations": [ " scale.", " chord.", " tempo.", " clef." ], "tags": [ "music" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -6.692070960998535, "mean_logprob": -3.3460354804992676, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 2, "sum_logprob": -2.7004072666168213, "mean_logprob": -1.3502036333084106, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 3, "sum_logprob": -12.314258575439453, "mean_logprob": -4.104752858479817, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 3, "sum_logprob": -12.103459358215332, "mean_logprob": -4.034486452738444, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 1.995831847190857, "context_truncated": false, "elapsed_seconds": 0.029503273000955232 }, { "id": "world_knowledge-031", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 0.0, "item_elo": 900, "context": "The blood vessels that generally carry blood away from the heart are", "continuations": [ " capillaries only.", " lymph nodes.", " arteries.", " alveoli." ], "tags": [ "biology" ], "prediction": 1, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -17.37689208984375, "mean_logprob": -3.47537841796875, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 3, "sum_logprob": -9.757102012634277, "mean_logprob": -3.2523673375447593, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 2, "sum_logprob": -7.300205230712891, "mean_logprob": -3.6501026153564453, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 4, "sum_logprob": -13.829465866088867, "mean_logprob": -3.457366466522217, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 0.20499912897745753, "context_truncated": false, "elapsed_seconds": 0.055547699999806355 }, { "id": "world_knowledge-032", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The decimal number ten is written in binary as", "continuations": [ " 1001.", " 1111.", " 1100.", " 1010." ], "tags": [ "computing" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -7.5533013343811035, "mean_logprob": -1.2588835557301838, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 6, "sum_logprob": -14.360594749450684, "mean_logprob": -2.3934324582417807, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 6, "sum_logprob": -9.910626411437988, "mean_logprob": -1.651771068572998, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 6, "sum_logprob": -4.08213472366333, "mean_logprob": -0.6803557872772217, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 0.5785277684529622, "context_truncated": false, "elapsed_seconds": 0.0371820000000298 }, { "id": "world_knowledge-033", "category": "world_knowledge", "difficulty": "medium", "category_weight": 1.1, "difficulty_weight": 1.5, "point_weight": 1.6500000000000001, "earned_points": 1.6500000000000001, "item_elo": 900, "context": "The mountain range commonly used as part of the boundary between Europe and Asia is the", "continuations": [ " Ural Mountains.", " Andes.", " Alps.", " Atlas Mountains." ], "tags": [ "geography" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -5.6374831199646, "mean_logprob": -1.40937077999115, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 3, "sum_logprob": -6.830323219299316, "mean_logprob": -2.2767744064331055, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 3, "sum_logprob": -4.304528713226318, "mean_logprob": -1.4348429044087727, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 4, "sum_logprob": -7.810175895690918, "mean_logprob": -1.9525439739227295, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.02547212441762281, "context_truncated": false, "elapsed_seconds": 0.026710719999755383 }, { "id": "world_knowledge-034", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "The principle that no two electrons in an atom can share all four quantum numbers is the", "continuations": [ " Doppler effect.", " Pauli exclusion principle.", " uncertainty of measurement.", " Hubble law." ], "tags": [ "physics" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -14.631590843200684, "mean_logprob": -2.9263181686401367, "total_sequence_tokens": 22 }, { "index": 1, "tokens": 5, "sum_logprob": -6.522121429443359, "mean_logprob": -1.3044242858886719, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 4, "sum_logprob": -27.918291091918945, "mean_logprob": -6.979572772979736, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 4, "sum_logprob": -26.616729736328125, "mean_logprob": -6.654182434082031, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 1.6218938827514648, "context_truncated": false, "elapsed_seconds": 0.027726984000764787 }, { "id": "world_knowledge-035", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "The 1648 agreements often associated with the modern state system are known as the Peace of", "continuations": [ " Augsburg.", " Paris.", " Westphalia.", " Nijmegen." ], "tags": [ "history" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -4.747247695922852, "mean_logprob": -1.186811923980713, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 2, "sum_logprob": -3.567901849746704, "mean_logprob": -1.783950924873352, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 4, "sum_logprob": -1.6582977771759033, "mean_logprob": -0.41457444429397583, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 5, "sum_logprob": -6.829130172729492, "mean_logprob": -1.3658260345458983, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.7722374796867371, "context_truncated": false, "elapsed_seconds": 0.02695665899955202 }, { "id": "world_knowledge-036", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "In linguistics, the smallest unit that carries meaning is a", "continuations": [ " phoneme in every case.", " clause.", " diphthong.", " morpheme." ], "tags": [ "linguistics" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -26.508514404296875, "mean_logprob": -4.4180857340494795, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 2, "sum_logprob": -6.442619800567627, "mean_logprob": -3.2213099002838135, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 5, "sum_logprob": -12.539463996887207, "mean_logprob": -2.5078927993774416, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 4, "sum_logprob": -4.539055347442627, "mean_logprob": -1.1347638368606567, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 1.3731289625167848, "context_truncated": false, "elapsed_seconds": 0.02788937600053032 }, { "id": "world_knowledge-037", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "The enzyme that unwinds the DNA double helix during replication is", "continuations": [ " helicase.", " amylase.", " lipase.", " pepsin." ], "tags": [ "biology" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -7.697218418121338, "mean_logprob": -1.9243046045303345, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 4, "sum_logprob": -11.000181198120117, "mean_logprob": -2.7500452995300293, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 3, "sum_logprob": -12.166723251342773, "mean_logprob": -4.055574417114258, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 3, "sum_logprob": -11.399234771728516, "mean_logprob": -3.7997449239095054, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 0.8257406949996948, "context_truncated": false, "elapsed_seconds": 0.025740410999787855 }, { "id": "world_knowledge-038", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "The economic concept describing the value of the best forgone alternative is", "continuations": [ " sunk cost.", " opportunity cost.", " marginal revenue.", " price elasticity." ], "tags": [ "economics" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -15.04764461517334, "mean_logprob": -3.761911153793335, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 3, "sum_logprob": -7.921466827392578, "mean_logprob": -2.640488942464193, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 3, "sum_logprob": -16.147680282592773, "mean_logprob": -5.382560094197591, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 4, "sum_logprob": -15.714448928833008, "mean_logprob": -3.928612232208252, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 1.121422211329142, "context_truncated": false, "elapsed_seconds": 0.07778051700006472 }, { "id": "world_knowledge-039", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "The literary device in which an absent or imaginary person is directly addressed is", "continuations": [ " metonymy.", " synecdoche.", " apostrophe.", " enjambment." ], "tags": [ "literature" ], "prediction": 1, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -9.08511734008789, "mean_logprob": -2.2712793350219727, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 5, "sum_logprob": -10.178845405578613, "mean_logprob": -2.0357690811157227, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 5, "sum_logprob": -12.251620292663574, "mean_logprob": -2.450324058532715, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 5, "sum_logprob": -13.08072280883789, "mean_logprob": -2.616144561767578, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.2355102539062499, "context_truncated": false, "elapsed_seconds": 0.027328506999765523 }, { "id": "world_knowledge-040", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "The ocean current that carries warm water across the North Atlantic is commonly called the", "continuations": [ " Humboldt Current.", " Benguela Current.", " Oyashio Current.", " Gulf Stream." ], "tags": [ "earth_science" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -7.287515640258789, "mean_logprob": -1.2145859400431316, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 6, "sum_logprob": -9.331193923950195, "mean_logprob": -1.5551989873250325, "total_sequence_tokens": 21 }, { "index": 2, "tokens": 6, "sum_logprob": -22.413164138793945, "mean_logprob": -3.7355273564656577, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 3, "sum_logprob": -0.9502784609794617, "mean_logprob": -0.31675948699315387, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 0.8978264530499778, "context_truncated": false, "elapsed_seconds": 0.027548702000785852 }, { "id": "world_knowledge-041", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "The alloy traditionally made primarily from copper and tin is", "continuations": [ " bronze.", " brass.", " steel.", " pewter." ], "tags": [ "materials" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -10.973139762878418, "mean_logprob": -3.657713254292806, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 3, "sum_logprob": -8.976274490356445, "mean_logprob": -2.992091496785482, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 2, "sum_logprob": -10.525206565856934, "mean_logprob": -5.262603282928467, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 4, "sum_logprob": -15.067255020141602, "mean_logprob": -3.7668137550354004, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 0.6656217575073242, "context_truncated": false, "elapsed_seconds": 0.02840788299909036 }, { "id": "world_knowledge-042", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "The mathematical constant that is the base of natural logarithms is", "continuations": [ " pi.", " e.", " phi.", " tau." ], "tags": [ "mathematics" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -6.910679340362549, "mean_logprob": -3.4553396701812744, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 2, "sum_logprob": -3.696976900100708, "mean_logprob": -1.848488450050354, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 2, "sum_logprob": -12.537050247192383, "mean_logprob": -6.268525123596191, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 2, "sum_logprob": -18.091705322265625, "mean_logprob": -9.045852661132812, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 1.6068512201309204, "context_truncated": false, "elapsed_seconds": 0.02433855900017079 }, { "id": "world_knowledge-043", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "In databases, a key that uniquely identifies each row in a table is a", "continuations": [ " foreign schema.", " null index.", " primary key.", " stored procedure." ], "tags": [ "computing" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -24.418195724487305, "mean_logprob": -8.139398574829102, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 3, "sum_logprob": -12.958831787109375, "mean_logprob": -4.319610595703125, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 3, "sum_logprob": -3.394084930419922, "mean_logprob": -1.1313616434733074, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 3, "sum_logprob": -9.721197128295898, "mean_logprob": -3.2403990427652993, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 2.109037399291992, "context_truncated": false, "elapsed_seconds": 0.02499480299957213 }, { "id": "world_knowledge-044", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "The branch of philosophy concerned with the nature and limits of knowledge is", "continuations": [ " aesthetics.", " ethics.", " ontology alone.", " epistemology." ], "tags": [ "philosophy" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -12.635092735290527, "mean_logprob": -4.211697578430176, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 2, "sum_logprob": -12.695484161376953, "mean_logprob": -6.347742080688477, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 4, "sum_logprob": -20.721094131469727, "mean_logprob": -5.180273532867432, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 5, "sum_logprob": -7.288669586181641, "mean_logprob": -1.4577339172363282, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 2.753963661193848, "context_truncated": false, "elapsed_seconds": 0.0514239969998016 }, { "id": "world_knowledge-045", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "The element with atomic number 74 is", "continuations": [ " tungsten.", " titanium.", " tantalum.", " tellurium." ], "tags": [ "chemistry" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -9.828885078430176, "mean_logprob": -1.9657770156860352, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 3, "sum_logprob": -10.466218948364258, "mean_logprob": -3.4887396494547525, "total_sequence_tokens": 12 }, { "index": 2, "tokens": 5, "sum_logprob": -10.116809844970703, "mean_logprob": -2.0233619689941404, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 4, "sum_logprob": -11.358242988586426, "mean_logprob": -2.8395607471466064, "total_sequence_tokens": 13 } ], "best_vs_second_margin": 0.0575849533081052, "context_truncated": false, "elapsed_seconds": 0.027971211999101797 }, { "id": "world_knowledge-046", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "The ancient writing system used for the Mycenaean Greek language is", "continuations": [ " cuneiform.", " Linear B.", " ogham.", " hieratic." ], "tags": [ "history" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -8.766053199768066, "mean_logprob": -2.1915132999420166, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 3, "sum_logprob": -7.610755920410156, "mean_logprob": -2.5369186401367188, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 4, "sum_logprob": -20.13168716430664, "mean_logprob": -5.03292179107666, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 4, "sum_logprob": -12.041725158691406, "mean_logprob": -3.0104312896728516, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.34540534019470215, "context_truncated": false, "elapsed_seconds": 0.026163692000409355 }, { "id": "world_knowledge-047", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "A star's spectral class in the sequence OBAFGKM is primarily related to its", "continuations": [ " distance from Earth.", " number of planets.", " surface temperature.", " age alone." ], "tags": [ "astronomy" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -8.229168891906738, "mean_logprob": -2.0572922229766846, "total_sequence_tokens": 22 }, { "index": 1, "tokens": 4, "sum_logprob": -18.440418243408203, "mean_logprob": -4.610104560852051, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 3, "sum_logprob": -5.304070949554443, "mean_logprob": -1.7680236498514812, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 3, "sum_logprob": -14.59379768371582, "mean_logprob": -4.864599227905273, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 0.2892685731252034, "context_truncated": false, "elapsed_seconds": 0.02666007100015122 }, { "id": "world_knowledge-048", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "The grammatical case typically marking direct address is the", "continuations": [ " accusative.", " genitive.", " instrumental.", " vocative." ], "tags": [ "linguistics" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -15.903148651123047, "mean_logprob": -3.9757871627807617, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 3, "sum_logprob": -15.181949615478516, "mean_logprob": -5.060649871826172, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 3, "sum_logprob": -17.94576072692871, "mean_logprob": -5.98192024230957, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 3, "sum_logprob": -15.720349311828613, "mean_logprob": -5.240116437276204, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 1.0848627090454102, "context_truncated": false, "elapsed_seconds": 0.03568337000069732 }, { "id": "world_knowledge-049", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 2.475, "item_elo": 1050, "context": "The boundary around a black hole beyond which light cannot escape is the", "continuations": [ " event horizon.", " heliopause.", " Roche limit.", " photosphere." ], "tags": [ "physics" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -1.2948393821716309, "mean_logprob": -0.43161312739054364, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 5, "sum_logprob": -7.611450672149658, "mean_logprob": -1.5222901344299316, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 4, "sum_logprob": -10.070302963256836, "mean_logprob": -2.517575740814209, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 3, "sum_logprob": -11.192702293395996, "mean_logprob": -3.730900764465332, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 1.0906770070393879, "context_truncated": false, "elapsed_seconds": 0.030300678001367487 }, { "id": "world_knowledge-050", "category": "world_knowledge", "difficulty": "hard", "category_weight": 1.1, "difficulty_weight": 2.25, "point_weight": 2.475, "earned_points": 0.0, "item_elo": 1050, "context": "In rhetoric, an argument that attacks the person rather than the claim is called", "continuations": [ " reductio ad absurdum.", " ad hominem.", " modus ponens.", " false equivalence only." ], "tags": [ "rhetoric" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -8.812560081481934, "mean_logprob": -1.1015700101852417, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 5, "sum_logprob": -9.008734703063965, "mean_logprob": -1.801746940612793, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 6, "sum_logprob": -17.020626068115234, "mean_logprob": -2.836771011352539, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 4, "sum_logprob": -30.69535255432129, "mean_logprob": -7.673838138580322, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 0.7001769304275514, "context_truncated": false, "elapsed_seconds": 0.035687636000147904 }, { "id": "context_tracking-001", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 1.2, "item_elo": 850, "context": "On the shelf, the red box contains tea and the blue box contains rice. The tea is in", "continuations": [ " the red box.", " the blue box.", " both boxes.", " neither box." ], "tags": [ "binding" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -6.783926486968994, "mean_logprob": -1.6959816217422485, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 4, "sum_logprob": -7.442936897277832, "mean_logprob": -1.860734224319458, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 3, "sum_logprob": -11.799026489257812, "mean_logprob": -3.933008829752604, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 3, "sum_logprob": -19.635343551635742, "mean_logprob": -6.545114517211914, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.16475260257720947, "context_truncated": false, "elapsed_seconds": 0.028946387999894796 }, { "id": "context_tracking-002", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 1.2, "item_elo": 850, "context": "Ivo chose the violin, while Mara chose the flute. Mara's instrument is", "continuations": [ " the violin.", " unspecified.", " the drum.", " the flute." ], "tags": [ "binding" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -7.989151477813721, "mean_logprob": -1.9972878694534302, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 2, "sum_logprob": -13.466575622558594, "mean_logprob": -6.733287811279297, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 3, "sum_logprob": -11.59686279296875, "mean_logprob": -3.8656209309895835, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 4, "sum_logprob": -5.811376094818115, "mean_logprob": -1.4528440237045288, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.5444438457489014, "context_truncated": false, "elapsed_seconds": 0.02690631699988444 }, { "id": "context_tracking-003", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 1.2, "item_elo": 850, "context": "The morning bus stops at Oak Street. The evening bus stops at Pine Street. The evening stop is", "continuations": [ " Oak Street.", " Maple Street.", " Pine Street.", " Cedar Street." ], "tags": [ "binding" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -3.7684435844421387, "mean_logprob": -1.256147861480713, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 4, "sum_logprob": -8.26771354675293, "mean_logprob": -2.0669283866882324, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 4, "sum_logprob": -4.313518524169922, "mean_logprob": -1.0783796310424805, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 5, "sum_logprob": -9.800602912902832, "mean_logprob": -1.9601205825805663, "total_sequence_tokens": 26 } ], "best_vs_second_margin": 0.17776823043823242, "context_truncated": false, "elapsed_seconds": 0.029487604000678402 }, { "id": "context_tracking-004", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 0.0, "item_elo": 850, "context": "Nia's badge number is 27, and Sol's badge number is 41. Badge 41 belongs to", "continuations": [ " Nia.", " both people.", " nobody.", " Sol." ], "tags": [ "binding" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -8.971212387084961, "mean_logprob": -2.9904041290283203, "total_sequence_tokens": 34 }, { "index": 1, "tokens": 3, "sum_logprob": -16.68063735961914, "mean_logprob": -5.56021245320638, "total_sequence_tokens": 34 }, { "index": 2, "tokens": 3, "sum_logprob": -14.08039379119873, "mean_logprob": -4.693464597066243, "total_sequence_tokens": 34 }, { "index": 3, "tokens": 2, "sum_logprob": -7.419593334197998, "mean_logprob": -3.709796667098999, "total_sequence_tokens": 33 } ], "best_vs_second_margin": 0.7193925380706787, "context_truncated": false, "elapsed_seconds": 0.0441296480003075 }, { "id": "context_tracking-005", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 0.0, "item_elo": 850, "context": "The north garden grows mint. The south garden grows basil. Mint grows in", "continuations": [ " the north garden.", " the south garden.", " both gardens.", " the greenhouse." ], "tags": [ "binding" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -4.372170448303223, "mean_logprob": -1.0930426120758057, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 4, "sum_logprob": -4.286496639251709, "mean_logprob": -1.0716241598129272, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 3, "sum_logprob": -6.398435592651367, "mean_logprob": -2.1328118642171225, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 3, "sum_logprob": -5.518257141113281, "mean_logprob": -1.8394190470377605, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.021418452262878418, "context_truncated": false, "elapsed_seconds": 0.07577670899991062 }, { "id": "context_tracking-006", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 0.0, "item_elo": 850, "context": "Room A is painted yellow; Room B is painted green. The green room is", "continuations": [ " Room A.", " Room B.", " both rooms.", " Room C." ], "tags": [ "binding" ], "prediction": 3, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -19.31879425048828, "mean_logprob": -6.439598083496094, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 3, "sum_logprob": -20.013511657714844, "mean_logprob": -6.671170552571614, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 3, "sum_logprob": -15.426803588867188, "mean_logprob": -5.1422678629557295, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 3, "sum_logprob": -14.464043617248535, "mean_logprob": -4.821347872416179, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.3209199905395508, "context_truncated": false, "elapsed_seconds": 0.031925275001412956 }, { "id": "context_tracking-007", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 1.2, "item_elo": 850, "context": "At noon, the cafe serves soup. At night, it serves noodles. The night menu features", "continuations": [ " soup.", " salad.", " noodles.", " bread only." ], "tags": [ "binding" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -15.040764808654785, "mean_logprob": -7.520382404327393, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 3, "sum_logprob": -18.535228729248047, "mean_logprob": -6.178409576416016, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 4, "sum_logprob": -14.340347290039062, "mean_logprob": -3.5850868225097656, "total_sequence_tokens": 27 }, { "index": 3, "tokens": 3, "sum_logprob": -27.44425392150879, "mean_logprob": -9.14808464050293, "total_sequence_tokens": 26 } ], "best_vs_second_margin": 2.59332275390625, "context_truncated": false, "elapsed_seconds": 0.08141225300096266 }, { "id": "context_tracking-008", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 1.2, "item_elo": 850, "context": "Paz owns the bicycle, and Ren owns the scooter. The scooter belongs to", "continuations": [ " Paz.", " both Paz and Ren.", " neither person.", " Ren." ], "tags": [ "binding" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -8.017608642578125, "mean_logprob": -2.6725362141927085, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 6, "sum_logprob": -14.909689903259277, "mean_logprob": -2.4849483172098794, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 3, "sum_logprob": -17.485139846801758, "mean_logprob": -5.828379948933919, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 2, "sum_logprob": -3.9368906021118164, "mean_logprob": -1.9684453010559082, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.5165030161539712, "context_truncated": false, "elapsed_seconds": 0.035989186999358935 }, { "id": "context_tracking-009", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 1.2, "item_elo": 850, "context": "The silver key opens the attic. The copper key opens the cellar. The attic key is", "continuations": [ " silver.", " copper.", " wooden.", " black." ], "tags": [ "binding" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -7.160383701324463, "mean_logprob": -3.5801918506622314, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 2, "sum_logprob": -9.434481620788574, "mean_logprob": -4.717240810394287, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 2, "sum_logprob": -9.718422889709473, "mean_logprob": -4.859211444854736, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 2, "sum_logprob": -8.321786880493164, "mean_logprob": -4.160893440246582, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.5807015895843506, "context_truncated": false, "elapsed_seconds": 0.03191114499895775 }, { "id": "context_tracking-010", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 1.2, "item_elo": 850, "context": "Toma ordered pears, while Eda ordered plums. Eda ordered", "continuations": [ " pears.", " plums.", " peaches.", " both fruits." ], "tags": [ "binding" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -3.681889057159424, "mean_logprob": -1.2272963523864746, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 3, "sum_logprob": -3.5526955127716064, "mean_logprob": -1.1842318375905354, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 3, "sum_logprob": -3.6570119857788086, "mean_logprob": -1.2190039952596028, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 3, "sum_logprob": -8.857694625854492, "mean_logprob": -2.9525648752848306, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.03477215766906738, "context_truncated": false, "elapsed_seconds": 0.029180496001572465 }, { "id": "context_tracking-011", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 0.0, "item_elo": 850, "context": "The first train goes to Linz. The second goes to Graz. The second destination is", "continuations": [ " Linz.", " Vienna.", " Graz.", " Salzburg." ], "tags": [ "binding" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -3.7210230827331543, "mean_logprob": -1.240341027577718, "total_sequence_tokens": 22 }, { "index": 1, "tokens": 3, "sum_logprob": -6.172287464141846, "mean_logprob": -2.0574291547139487, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 3, "sum_logprob": -5.512436866760254, "mean_logprob": -1.8374789555867512, "total_sequence_tokens": 22 }, { "index": 3, "tokens": 4, "sum_logprob": -10.283622741699219, "mean_logprob": -2.5709056854248047, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.5971379280090332, "context_truncated": false, "elapsed_seconds": 0.03351690399904328 }, { "id": "context_tracking-012", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 0.0, "item_elo": 850, "context": "File K is dated March, and File M is dated June. The June file is", "continuations": [ " File K.", " both files.", " undated.", " File M." ], "tags": [ "binding" ], "prediction": 2, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -16.309968948364258, "mean_logprob": -5.436656316121419, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 3, "sum_logprob": -18.492210388183594, "mean_logprob": -6.164070129394531, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 3, "sum_logprob": -9.313602447509766, "mean_logprob": -3.104534149169922, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 3, "sum_logprob": -19.71459197998047, "mean_logprob": -6.571530659993489, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 2.332122166951497, "context_truncated": false, "elapsed_seconds": 0.030803358000412118 }, { "id": "context_tracking-013", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 1.2, "item_elo": 850, "context": "Luma wears a square pin. Oren wears a round pin. Luma's pin is", "continuations": [ " square.", " round.", " triangular.", " missing." ], "tags": [ "binding" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -6.6852593421936035, "mean_logprob": -3.3426296710968018, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 2, "sum_logprob": -8.037212371826172, "mean_logprob": -4.018606185913086, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 2, "sum_logprob": -8.829096794128418, "mean_logprob": -4.414548397064209, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 2, "sum_logprob": -7.15470027923584, "mean_logprob": -3.57735013961792, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 0.23472046852111816, "context_truncated": false, "elapsed_seconds": 0.026890174000072875 }, { "id": "context_tracking-014", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 0.0, "item_elo": 850, "context": "The code for apple is 6. The code for orange is 9. Code 9 means", "continuations": [ " apple.", " orange.", " pear.", " fruit in general." ], "tags": [ "binding" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -12.063654899597168, "mean_logprob": -6.031827449798584, "total_sequence_tokens": 22 }, { "index": 1, "tokens": 2, "sum_logprob": -13.191564559936523, "mean_logprob": -6.595782279968262, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 3, "sum_logprob": -16.624719619750977, "mean_logprob": -5.541573206583659, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 4, "sum_logprob": -25.945880889892578, "mean_logprob": -6.4864702224731445, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 0.49025424321492483, "context_truncated": false, "elapsed_seconds": 0.027514989000337664 }, { "id": "context_tracking-015", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 1.2, "item_elo": 850, "context": "Kira arrived before Bo, and Bo arrived before Luis. The last to arrive was", "continuations": [ " Kira.", " Bo.", " Luis.", " nobody." ], "tags": [ "order" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -6.463204383850098, "mean_logprob": -2.1544014612833657, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 2, "sum_logprob": -6.712535858154297, "mean_logprob": -3.3562679290771484, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 3, "sum_logprob": -3.134509563446045, "mean_logprob": -1.0448365211486816, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 3, "sum_logprob": -11.898235321044922, "mean_logprob": -3.966078440348307, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 1.109564940134684, "context_truncated": false, "elapsed_seconds": 0.027884559000085574 }, { "id": "context_tracking-016", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 0.0, "item_elo": 850, "context": "The small parcel weighs two kilos. The large parcel weighs five. The heavier parcel is", "continuations": [ " the small parcel.", " both equally.", " unknown.", " the large parcel." ], "tags": [ "comparison" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -12.038433074951172, "mean_logprob": -2.4076866149902343, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 3, "sum_logprob": -22.769210815429688, "mean_logprob": -7.5897369384765625, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 2, "sum_logprob": -11.203941345214844, "mean_logprob": -5.601970672607422, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 5, "sum_logprob": -13.794227600097656, "mean_logprob": -2.7588455200195314, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.35115890502929714, "context_truncated": false, "elapsed_seconds": 0.02827097000044887 }, { "id": "context_tracking-017", "category": "context_tracking", "difficulty": "easy", "category_weight": 1.2, "difficulty_weight": 1.0, "point_weight": 1.2, "earned_points": 0.0, "item_elo": 850, "context": "Tuesday's password is comet. Wednesday's password is lantern. On Tuesday, the password is", "continuations": [ " comet.", " lantern.", " Tuesday.", " Wednesday." ], "tags": [ "binding" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -7.41854190826416, "mean_logprob": -2.47284730275472, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 4, "sum_logprob": -4.859280586242676, "mean_logprob": -1.214820146560669, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 2, "sum_logprob": -8.315832138061523, "mean_logprob": -4.157916069030762, "total_sequence_tokens": 26 }, { "index": 3, "tokens": 2, "sum_logprob": -8.089353561401367, "mean_logprob": -4.044676780700684, "total_sequence_tokens": 26 } ], "best_vs_second_margin": 1.2580271561940513, "context_truncated": false, "elapsed_seconds": 0.03253696600040712 }, { "id": "context_tracking-018", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 0.0, "item_elo": 950, "context": "Three couriers carry different folders: Aya has violet, Ben has white, and Cy has black. The white folder is carried by", "continuations": [ " Aya.", " Ben.", " Cy.", " no courier." ], "tags": [ "multi_binding" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -4.21363639831543, "mean_logprob": -1.4045454661051433, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 2, "sum_logprob": -6.199267864227295, "mean_logprob": -3.0996339321136475, "total_sequence_tokens": 30 }, { "index": 2, "tokens": 2, "sum_logprob": -7.0412163734436035, "mean_logprob": -3.5206081867218018, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 4, "sum_logprob": -14.700834274291992, "mean_logprob": -3.675208568572998, "total_sequence_tokens": 32 } ], "best_vs_second_margin": 1.6950884660085042, "context_truncated": false, "elapsed_seconds": 0.034491018999688094 }, { "id": "context_tracking-019", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "The gallery opens after the cafe but before the theater. The theater opens after the library. Among the gallery, cafe, and theater, the earliest is", "continuations": [ " the gallery.", " the theater.", " the cafe.", " impossible to know." ], "tags": [ "order" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -7.34194278717041, "mean_logprob": -2.4473142623901367, "total_sequence_tokens": 35 }, { "index": 1, "tokens": 3, "sum_logprob": -6.660017967224121, "mean_logprob": -2.220005989074707, "total_sequence_tokens": 35 }, { "index": 2, "tokens": 4, "sum_logprob": -5.666388034820557, "mean_logprob": -1.4165970087051392, "total_sequence_tokens": 36 }, { "index": 3, "tokens": 4, "sum_logprob": -20.071624755859375, "mean_logprob": -5.017906188964844, "total_sequence_tokens": 36 } ], "best_vs_second_margin": 0.8034089803695679, "context_truncated": false, "elapsed_seconds": 0.04306272899884789 }, { "id": "context_tracking-020", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "A token moves from Box A to Box C, then from Box C to Box B. Its final location is", "continuations": [ " Box A.", " Box C.", " every box.", " Box B." ], "tags": [ "state_update" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -2.557363986968994, "mean_logprob": -0.852454662322998, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 3, "sum_logprob": -5.336918354034424, "mean_logprob": -1.7789727846781414, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 3, "sum_logprob": -21.708280563354492, "mean_logprob": -7.236093521118164, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 3, "sum_logprob": -2.3088560104370117, "mean_logprob": -0.7696186701456705, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.08283599217732751, "context_truncated": false, "elapsed_seconds": 0.025915561000147136 }, { "id": "context_tracking-021", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "Rin lends her map to Dev. Dev then gives the map back to Rin. At the end, the map is with", "continuations": [ " Rin.", " Dev.", " both of them.", " nobody." ], "tags": [ "state_update" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -2.3433592319488525, "mean_logprob": -0.7811197439829508, "total_sequence_tokens": 30 }, { "index": 1, "tokens": 2, "sum_logprob": -4.1370415687561035, "mean_logprob": -2.0685207843780518, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 4, "sum_logprob": -10.318612098693848, "mean_logprob": -2.579653024673462, "total_sequence_tokens": 31 }, { "index": 3, "tokens": 3, "sum_logprob": -17.195390701293945, "mean_logprob": -5.731796900431315, "total_sequence_tokens": 30 } ], "best_vs_second_margin": 1.287401040395101, "context_truncated": false, "elapsed_seconds": 0.02816087800056266 }, { "id": "context_tracking-022", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 0.0, "item_elo": 950, "context": "The amber team scored below the teal team but above the gray team. The highest-scoring of these teams was", "continuations": [ " amber.", " teal.", " gray.", " tied." ], "tags": [ "order" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -11.838001251220703, "mean_logprob": -3.946000417073568, "total_sequence_tokens": 28 }, { "index": 1, "tokens": 3, "sum_logprob": -13.28606128692627, "mean_logprob": -4.42868709564209, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 2, "sum_logprob": -12.694733619689941, "mean_logprob": -6.347366809844971, "total_sequence_tokens": 27 }, { "index": 3, "tokens": 2, "sum_logprob": -13.777946472167969, "mean_logprob": -6.888973236083984, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.482686678568522, "context_truncated": false, "elapsed_seconds": 0.029178336000768468 }, { "id": "context_tracking-023", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "Ava's shift is Monday, Bram's is Thursday, and Cato's is Tuesday. The person working between Ava and Bram in the week is", "continuations": [ " Ava.", " Bram.", " Cato.", " nobody." ], "tags": [ "calendar" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -6.949986457824707, "mean_logprob": -2.316662152608236, "total_sequence_tokens": 39 }, { "index": 1, "tokens": 3, "sum_logprob": -6.109921932220459, "mean_logprob": -2.0366406440734863, "total_sequence_tokens": 39 }, { "index": 2, "tokens": 3, "sum_logprob": -4.568081378936768, "mean_logprob": -1.5226937929789226, "total_sequence_tokens": 39 }, { "index": 3, "tokens": 3, "sum_logprob": -12.695931434631348, "mean_logprob": -4.231977144877116, "total_sequence_tokens": 39 } ], "best_vs_second_margin": 0.5139468510945637, "context_truncated": false, "elapsed_seconds": 0.046897251000700635 }, { "id": "context_tracking-024", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "The note began under the lamp. Jo moved it to the desk, and Uma moved it from the desk into a drawer. The note ended", "continuations": [ " under the lamp.", " on the desk.", " with Jo.", " in the drawer." ], "tags": [ "state_update" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -6.655835151672363, "mean_logprob": -1.6639587879180908, "total_sequence_tokens": 34 }, { "index": 1, "tokens": 4, "sum_logprob": -10.382094383239746, "mean_logprob": -2.5955235958099365, "total_sequence_tokens": 34 }, { "index": 2, "tokens": 3, "sum_logprob": -11.422002792358398, "mean_logprob": -3.8073342641194663, "total_sequence_tokens": 33 }, { "index": 3, "tokens": 5, "sum_logprob": -7.003646373748779, "mean_logprob": -1.400729274749756, "total_sequence_tokens": 35 } ], "best_vs_second_margin": 0.26322951316833487, "context_truncated": false, "elapsed_seconds": 0.047904680999636184 }, { "id": "context_tracking-025", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "The labels are reassigned: north now means left, south means right, east means up, and west means down. Under this rule, north means", "continuations": [ " left.", " right.", " up.", " down." ], "tags": [ "remapping" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -4.648298263549805, "mean_logprob": -2.3241491317749023, "total_sequence_tokens": 32 }, { "index": 1, "tokens": 2, "sum_logprob": -5.928032398223877, "mean_logprob": -2.9640161991119385, "total_sequence_tokens": 32 }, { "index": 2, "tokens": 2, "sum_logprob": -5.894533634185791, "mean_logprob": -2.9472668170928955, "total_sequence_tokens": 32 }, { "index": 3, "tokens": 2, "sum_logprob": -5.785933494567871, "mean_logprob": -2.8929667472839355, "total_sequence_tokens": 32 } ], "best_vs_second_margin": 0.5688176155090332, "context_truncated": false, "elapsed_seconds": 0.034175955999671714 }, { "id": "context_tracking-026", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 0.0, "item_elo": 950, "context": "The bronze cup is taller than the glass cup. The glass cup is taller than the clay cup. The shortest is", "continuations": [ " bronze.", " clay.", " glass.", " not specified." ], "tags": [ "transitive" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -9.209604263305664, "mean_logprob": -3.0698680877685547, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 2, "sum_logprob": -10.724428176879883, "mean_logprob": -5.362214088439941, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 2, "sum_logprob": -8.372103691101074, "mean_logprob": -4.186051845550537, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 3, "sum_logprob": -10.977124214172363, "mean_logprob": -3.659041404724121, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.5891733169555664, "context_truncated": false, "elapsed_seconds": 0.02894302099957713 }, { "id": "context_tracking-027", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 0.0, "item_elo": 950, "context": "Nell assigns fox to 14, owl to 22, and hare to 31. The animal assigned 31 is", "continuations": [ " fox.", " owl.", " hare.", " none." ], "tags": [ "binding" ], "prediction": 1, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -10.879972457885742, "mean_logprob": -5.439986228942871, "total_sequence_tokens": 34 }, { "index": 1, "tokens": 3, "sum_logprob": -10.412038803100586, "mean_logprob": -3.470679601033529, "total_sequence_tokens": 35 }, { "index": 2, "tokens": 3, "sum_logprob": -11.36612319946289, "mean_logprob": -3.788707733154297, "total_sequence_tokens": 35 }, { "index": 3, "tokens": 2, "sum_logprob": -14.26858139038086, "mean_logprob": -7.13429069519043, "total_sequence_tokens": 34 } ], "best_vs_second_margin": 0.3180281321207681, "context_truncated": false, "elapsed_seconds": 0.03798832400025276 }, { "id": "context_tracking-028", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 0.0, "item_elo": 950, "context": "A meeting moves from Room 3 to Room 8, then is moved back to its original room. It will occur in", "continuations": [ " Room 8.", " both rooms.", " no room.", " Room 3." ], "tags": [ "state_update" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -4.633544921875, "mean_logprob": -1.15838623046875, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 3, "sum_logprob": -10.276191711425781, "mean_logprob": -3.4253972371419272, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 3, "sum_logprob": -16.473928451538086, "mean_logprob": -5.491309483846028, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 4, "sum_logprob": -6.828451156616211, "mean_logprob": -1.7071127891540527, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.5487265586853027, "context_truncated": false, "elapsed_seconds": 0.03693265699985204 }, { "id": "context_tracking-029", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 0.0, "item_elo": 950, "context": "In a made-up calendar, Dax follows Rul, and Vem follows Dax. The day immediately after Rul is", "continuations": [ " Dax.", " Vem.", " Rul.", " unknown." ], "tags": [ "local_rules" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -7.311236381530762, "mean_logprob": -2.4370787938435874, "total_sequence_tokens": 30 }, { "index": 1, "tokens": 3, "sum_logprob": -6.153491973876953, "mean_logprob": -2.051163991292318, "total_sequence_tokens": 30 }, { "index": 2, "tokens": 3, "sum_logprob": -9.810539245605469, "mean_logprob": -3.2701797485351562, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 2, "sum_logprob": -7.88333797454834, "mean_logprob": -3.94166898727417, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.38591480255126953, "context_truncated": false, "elapsed_seconds": 0.03235356599907391 }, { "id": "context_tracking-030", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "Faye is older than Gus but younger than Hana. The oldest of the three is", "continuations": [ " Faye.", " Hana.", " Gus.", " tied." ], "tags": [ "transitive" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -6.905505180358887, "mean_logprob": -1.7263762950897217, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 3, "sum_logprob": -3.8858604431152344, "mean_logprob": -1.2952868143717449, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 3, "sum_logprob": -3.9375972747802734, "mean_logprob": -1.3125324249267578, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 2, "sum_logprob": -12.933774948120117, "mean_logprob": -6.466887474060059, "total_sequence_tokens": 22 } ], "best_vs_second_margin": 0.017245610555012947, "context_truncated": false, "elapsed_seconds": 0.04053015899989987 }, { "id": "context_tracking-031", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "The red route reaches Delta before Echo. The blue route reaches Echo before Delta. On the blue route, the first of those stops is", "continuations": [ " Delta.", " neither stop.", " Echo.", " both together." ], "tags": [ "conditional_binding" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -6.452212810516357, "mean_logprob": -3.2261064052581787, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 3, "sum_logprob": -18.530963897705078, "mean_logprob": -6.176987965901692, "total_sequence_tokens": 32 }, { "index": 2, "tokens": 3, "sum_logprob": -4.69672155380249, "mean_logprob": -1.5655738512674968, "total_sequence_tokens": 32 }, { "index": 3, "tokens": 3, "sum_logprob": -21.00954818725586, "mean_logprob": -7.003182729085286, "total_sequence_tokens": 32 } ], "best_vs_second_margin": 1.660532553990682, "context_truncated": false, "elapsed_seconds": 0.03073819699966407 }, { "id": "context_tracking-032", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "Iris gives the ring to Juno, Juno gives it to Kato, and Kato keeps it. The ring is finally with", "continuations": [ " Iris.", " Juno.", " everyone.", " Kato." ], "tags": [ "state_update" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -11.189102172851562, "mean_logprob": -3.729700724283854, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 3, "sum_logprob": -8.019768714904785, "mean_logprob": -2.673256238301595, "total_sequence_tokens": 31 }, { "index": 2, "tokens": 2, "sum_logprob": -10.497590065002441, "mean_logprob": -5.248795032501221, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 3, "sum_logprob": -7.808121681213379, "mean_logprob": -2.6027072270711265, "total_sequence_tokens": 31 } ], "best_vs_second_margin": 0.07054901123046875, "context_truncated": false, "elapsed_seconds": 0.026309938000849797 }, { "id": "context_tracking-033", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 1.7999999999999998, "item_elo": 950, "context": "For this report, green means urgent and red means routine. A green item is therefore", "continuations": [ " urgent.", " routine.", " canceled.", " unclassified." ], "tags": [ "remapping" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -8.035861015319824, "mean_logprob": -2.6786203384399414, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 2, "sum_logprob": -11.059852600097656, "mean_logprob": -5.529926300048828, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 4, "sum_logprob": -15.928162574768066, "mean_logprob": -3.9820406436920166, "total_sequence_tokens": 22 }, { "index": 3, "tokens": 4, "sum_logprob": -17.694698333740234, "mean_logprob": -4.423674583435059, "total_sequence_tokens": 22 } ], "best_vs_second_margin": 1.3034203052520752, "context_truncated": false, "elapsed_seconds": 0.026921026001218706 }, { "id": "context_tracking-034", "category": "context_tracking", "difficulty": "medium", "category_weight": 1.2, "difficulty_weight": 1.5, "point_weight": 1.7999999999999998, "earned_points": 0.0, "item_elo": 950, "context": "The oak is east of the pond. The bench is east of the oak. Relative to the pond, the bench is", "continuations": [ " west.", " east.", " north.", " at the same place." ], "tags": [ "spatial" ], "prediction": 3, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -8.630821228027344, "mean_logprob": -4.315410614013672, "total_sequence_tokens": 28 }, { "index": 1, "tokens": 2, "sum_logprob": -8.045670509338379, "mean_logprob": -4.0228352546691895, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 2, "sum_logprob": -7.104247093200684, "mean_logprob": -3.552123546600342, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 5, "sum_logprob": -17.643739700317383, "mean_logprob": -3.5287479400634765, "total_sequence_tokens": 31 } ], "best_vs_second_margin": 0.023375606536865323, "context_truncated": false, "elapsed_seconds": 0.028784793001250364 }, { "id": "context_tracking-035", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 0.0, "item_elo": 1100, "context": "Four files are ordered J, K, L, M. K is moved after M, and J is then moved between L and M. The resulting order is", "continuations": [ " J, L, M, K.", " L, M, J, K.", " L, J, M, K.", " M, L, J, K." ], "tags": [ "state_update" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -5.9900970458984375, "mean_logprob": -0.7487621307373047, "total_sequence_tokens": 40 }, { "index": 1, "tokens": 8, "sum_logprob": -8.535561561584473, "mean_logprob": -1.066945195198059, "total_sequence_tokens": 40 }, { "index": 2, "tokens": 8, "sum_logprob": -8.659443855285645, "mean_logprob": -1.0824304819107056, "total_sequence_tokens": 40 }, { "index": 3, "tokens": 8, "sum_logprob": -11.459959030151367, "mean_logprob": -1.432494878768921, "total_sequence_tokens": 40 } ], "best_vs_second_margin": 0.3181830644607544, "context_truncated": false, "elapsed_seconds": 0.046220570999139454 }, { "id": "context_tracking-036", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 0.0, "item_elo": 1100, "context": "In a codebook, tulip means north, north means copper, and copper means seven. Following the chain from tulip leads to", "continuations": [ " north only.", " tulip.", " copper only.", " seven." ], "tags": [ "multi_hop" ], "prediction": 1, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -17.177152633666992, "mean_logprob": -5.725717544555664, "total_sequence_tokens": 32 }, { "index": 1, "tokens": 4, "sum_logprob": -11.78298568725586, "mean_logprob": -2.945746421813965, "total_sequence_tokens": 33 }, { "index": 2, "tokens": 3, "sum_logprob": -15.028417587280273, "mean_logprob": -5.009472529093425, "total_sequence_tokens": 32 }, { "index": 3, "tokens": 2, "sum_logprob": -11.33794116973877, "mean_logprob": -5.668970584869385, "total_sequence_tokens": 31 } ], "best_vs_second_margin": 2.06372610727946, "context_truncated": false, "elapsed_seconds": 0.03617483699963486 }, { "id": "context_tracking-037", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 0.0, "item_elo": 1100, "context": "A cube starts in Bin R. It moves to S unless the flag is blue. The flag is blue, so it moves to T instead. Its final bin is", "continuations": [ " Bin T.", " Bin S.", " Bin R.", " unknown." ], "tags": [ "conditional_update" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -10.813809394836426, "mean_logprob": -3.604603131612142, "total_sequence_tokens": 36 }, { "index": 1, "tokens": 3, "sum_logprob": -7.571236610412598, "mean_logprob": -2.523745536804199, "total_sequence_tokens": 36 }, { "index": 2, "tokens": 3, "sum_logprob": -9.749302864074707, "mean_logprob": -3.249767621358236, "total_sequence_tokens": 36 }, { "index": 3, "tokens": 2, "sum_logprob": -6.8441290855407715, "mean_logprob": -3.4220645427703857, "total_sequence_tokens": 35 } ], "best_vs_second_margin": 0.7260220845540366, "context_truncated": false, "elapsed_seconds": 0.03776685400043789 }, { "id": "context_tracking-038", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 0.0, "item_elo": 1100, "context": "Mira stands left of Noor. Olin stands right of Noor but left of Peta. From left to right, the second person is", "continuations": [ " Mira.", " Noor.", " Olin.", " Peta." ], "tags": [ "spatial" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -4.994318962097168, "mean_logprob": -1.6647729873657227, "total_sequence_tokens": 33 }, { "index": 1, "tokens": 3, "sum_logprob": -4.311159610748291, "mean_logprob": -1.4370532035827637, "total_sequence_tokens": 33 }, { "index": 2, "tokens": 3, "sum_logprob": -2.769989252090454, "mean_logprob": -0.923329750696818, "total_sequence_tokens": 33 }, { "index": 3, "tokens": 3, "sum_logprob": -3.6630380153656006, "mean_logprob": -1.2210126717885335, "total_sequence_tokens": 33 } ], "best_vs_second_margin": 0.29768292109171546, "context_truncated": false, "elapsed_seconds": 0.034288531000129296 }, { "id": "context_tracking-039", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 2.6999999999999997, "item_elo": 1100, "context": "The archive uses rotating labels. On day one A means red; on day two A means blue; on day three A means gold. Today is day three, so A means", "continuations": [ " red.", " blue.", " gold.", " every color." ], "tags": [ "conditional_binding" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -4.996253967285156, "mean_logprob": -2.498126983642578, "total_sequence_tokens": 37 }, { "index": 1, "tokens": 2, "sum_logprob": -4.155964374542236, "mean_logprob": -2.077982187271118, "total_sequence_tokens": 37 }, { "index": 2, "tokens": 2, "sum_logprob": -0.3302699327468872, "mean_logprob": -0.1651349663734436, "total_sequence_tokens": 37 }, { "index": 3, "tokens": 3, "sum_logprob": -9.863689422607422, "mean_logprob": -3.287896474202474, "total_sequence_tokens": 38 } ], "best_vs_second_margin": 1.9128472208976746, "context_truncated": false, "elapsed_seconds": 0.04565987499881885 }, { "id": "context_tracking-040", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 0.0, "item_elo": 1100, "context": "A coin begins with Lio, passes to Mina, returns to Lio, then passes to Niko. Niko gives it to the person who held it immediately before him. The coin ends with", "continuations": [ " Mina.", " Niko.", " nobody.", " Lio." ], "tags": [ "state_update" ], "prediction": 1, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -1.7303788661956787, "mean_logprob": -0.5767929553985596, "total_sequence_tokens": 46 }, { "index": 1, "tokens": 4, "sum_logprob": -1.7619781494140625, "mean_logprob": -0.4404945373535156, "total_sequence_tokens": 47 }, { "index": 2, "tokens": 3, "sum_logprob": -12.21472454071045, "mean_logprob": -4.071574846903483, "total_sequence_tokens": 46 }, { "index": 3, "tokens": 3, "sum_logprob": -2.2031490802764893, "mean_logprob": -0.7343830267588297, "total_sequence_tokens": 46 } ], "best_vs_second_margin": 0.13629841804504395, "context_truncated": false, "elapsed_seconds": 0.04590733899931365 }, { "id": "context_tracking-041", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 2.6999999999999997, "item_elo": 1100, "context": "In a race, Uma finished before Vic. Wes finished after Xena but before Uma. The earliest among Uma, Vic, Wes, and Xena was", "continuations": [ " Xena.", " Wes.", " Uma.", " Vic." ], "tags": [ "transitive" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -2.8171894550323486, "mean_logprob": -0.9390631516774496, "total_sequence_tokens": 41 }, { "index": 1, "tokens": 3, "sum_logprob": -3.756889820098877, "mean_logprob": -1.2522966066996257, "total_sequence_tokens": 41 }, { "index": 2, "tokens": 3, "sum_logprob": -3.0636844635009766, "mean_logprob": -1.0212281545003254, "total_sequence_tokens": 41 }, { "index": 3, "tokens": 3, "sum_logprob": -2.9644932746887207, "mean_logprob": -0.9881644248962402, "total_sequence_tokens": 41 } ], "best_vs_second_margin": 0.04910127321879065, "context_truncated": false, "elapsed_seconds": 0.04572442499920726 }, { "id": "context_tracking-042", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 2.6999999999999997, "item_elo": 1100, "context": "A ledger maps 4 to pine, pine to west, and west to 19. Looking up 4 repeatedly until reaching a number returns", "continuations": [ " west.", " 19.", " pine.", " 4." ], "tags": [ "multi_hop" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -11.747142791748047, "mean_logprob": -5.873571395874023, "total_sequence_tokens": 33 }, { "index": 1, "tokens": 4, "sum_logprob": -4.2428412437438965, "mean_logprob": -1.0607103109359741, "total_sequence_tokens": 35 }, { "index": 2, "tokens": 2, "sum_logprob": -9.676970481872559, "mean_logprob": -4.838485240936279, "total_sequence_tokens": 33 }, { "index": 3, "tokens": 3, "sum_logprob": -6.3904032707214355, "mean_logprob": -2.130134423573812, "total_sequence_tokens": 34 } ], "best_vs_second_margin": 1.0694241126378379, "context_truncated": false, "elapsed_seconds": 0.03764528200008499 }, { "id": "context_tracking-043", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 0.0, "item_elo": 1100, "context": "The black card outranks the white card. The striped card ranks below white but above gray. The second-highest card is", "continuations": [ " black.", " striped.", " white.", " gray." ], "tags": [ "order" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -3.5012102127075195, "mean_logprob": -1.7506051063537598, "total_sequence_tokens": 30 }, { "index": 1, "tokens": 3, "sum_logprob": -8.898595809936523, "mean_logprob": -2.9661986033121743, "total_sequence_tokens": 31 }, { "index": 2, "tokens": 2, "sum_logprob": -4.047337532043457, "mean_logprob": -2.0236687660217285, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 2, "sum_logprob": -5.855133056640625, "mean_logprob": -2.9275665283203125, "total_sequence_tokens": 30 } ], "best_vs_second_margin": 0.27306365966796875, "context_truncated": false, "elapsed_seconds": 0.03667803600001207 }, { "id": "context_tracking-044", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 0.0, "item_elo": 1100, "context": "A marker is in Drawer A. Rule one moves it to B; rule two swaps the contents of B and C; rule three moves whatever is in C to D. The marker finishes in", "continuations": [ " Drawer A.", " Drawer B.", " Drawer C.", " Drawer D." ], "tags": [ "state_update" ], "prediction": 1, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -4.155341148376465, "mean_logprob": -1.0388352870941162, "total_sequence_tokens": 45 }, { "index": 1, "tokens": 4, "sum_logprob": -3.6743602752685547, "mean_logprob": -0.9185900688171387, "total_sequence_tokens": 45 }, { "index": 2, "tokens": 4, "sum_logprob": -5.7672014236450195, "mean_logprob": -1.4418003559112549, "total_sequence_tokens": 45 }, { "index": 3, "tokens": 4, "sum_logprob": -4.906957626342773, "mean_logprob": -1.2267394065856934, "total_sequence_tokens": 45 } ], "best_vs_second_margin": 0.12024521827697754, "context_truncated": false, "elapsed_seconds": 0.04519589900155552 }, { "id": "context_tracking-045", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 0.0, "item_elo": 1100, "context": "Q is north of R. S is east of Q. T is south of S and level with R. Relative to R, T is", "continuations": [ " east.", " west.", " north.", " at the same position." ], "tags": [ "spatial" ], "prediction": 3, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -10.444129943847656, "mean_logprob": -5.222064971923828, "total_sequence_tokens": 30 }, { "index": 1, "tokens": 2, "sum_logprob": -9.060394287109375, "mean_logprob": -4.5301971435546875, "total_sequence_tokens": 30 }, { "index": 2, "tokens": 2, "sum_logprob": -9.162068367004395, "mean_logprob": -4.581034183502197, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 5, "sum_logprob": -17.914993286132812, "mean_logprob": -3.5829986572265624, "total_sequence_tokens": 33 } ], "best_vs_second_margin": 0.9471984863281251, "context_truncated": false, "elapsed_seconds": 0.03621936600029585 }, { "id": "context_tracking-046", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 2.6999999999999997, "item_elo": 1100, "context": "There are three switches. Flipping X toggles Y; flipping Y toggles Z. Initially all are off. X is flipped once, then Y is flipped once. At the end, Z is", "continuations": [ " unchanged and off.", " on.", " removed.", " both on and off." ], "tags": [ "state_update" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -18.077299118041992, "mean_logprob": -4.519324779510498, "total_sequence_tokens": 47 }, { "index": 1, "tokens": 2, "sum_logprob": -1.6058751344680786, "mean_logprob": -0.8029375672340393, "total_sequence_tokens": 45 }, { "index": 2, "tokens": 2, "sum_logprob": -11.36464786529541, "mean_logprob": -5.682323932647705, "total_sequence_tokens": 45 }, { "index": 3, "tokens": 5, "sum_logprob": -8.00474739074707, "mean_logprob": -1.600949478149414, "total_sequence_tokens": 48 } ], "best_vs_second_margin": 0.7980119109153747, "context_truncated": false, "elapsed_seconds": 0.045616373001394095 }, { "id": "context_tracking-047", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 2.6999999999999997, "item_elo": 1100, "context": "A roster initially reads Arin, Bela, Ciro, Dena. Bela and Dena swap places; then the first and third names swap. The first name becomes", "continuations": [ " Arin.", " Dena.", " Ciro.", " Bela." ], "tags": [ "permutation" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -3.308192253112793, "mean_logprob": -1.1027307510375977, "total_sequence_tokens": 40 }, { "index": 1, "tokens": 3, "sum_logprob": -5.120464324951172, "mean_logprob": -1.7068214416503906, "total_sequence_tokens": 40 }, { "index": 2, "tokens": 3, "sum_logprob": -3.036705255508423, "mean_logprob": -1.0122350851694744, "total_sequence_tokens": 40 }, { "index": 3, "tokens": 3, "sum_logprob": -7.030072212219238, "mean_logprob": -2.3433574040730796, "total_sequence_tokens": 40 } ], "best_vs_second_margin": 0.0904956658681233, "context_truncated": false, "elapsed_seconds": 0.03578504999859433 }, { "id": "context_tracking-048", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 2.6999999999999997, "item_elo": 1100, "context": "On odd dates, Gate P leads to Hall 1; on even dates it leads to Hall 2. Gate Q does the opposite. The date is the 18th, and a visitor enters Gate Q, reaching", "continuations": [ " Hall 2.", " both halls.", " no hall.", " Hall 1." ], "tags": [ "conditional_binding" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -2.4496448040008545, "mean_logprob": -0.6124112010002136, "total_sequence_tokens": 52 }, { "index": 1, "tokens": 4, "sum_logprob": -12.987076759338379, "mean_logprob": -3.2467691898345947, "total_sequence_tokens": 52 }, { "index": 2, "tokens": 3, "sum_logprob": -17.982742309570312, "mean_logprob": -5.9942474365234375, "total_sequence_tokens": 51 }, { "index": 3, "tokens": 4, "sum_logprob": -0.7162995934486389, "mean_logprob": -0.17907489836215973, "total_sequence_tokens": 52 } ], "best_vs_second_margin": 0.4333363026380539, "context_truncated": false, "elapsed_seconds": 0.047340212999188225 }, { "id": "context_tracking-049", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 2.6999999999999997, "item_elo": 1100, "context": "Nora is assigned to Vega unless Ishan takes Vega; if he does, Nora takes Lyra. Ishan takes Vega. Nora is therefore assigned to", "continuations": [ " Lyra.", " Vega.", " both.", " neither." ], "tags": [ "conditional_binding" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -1.1825419664382935, "mean_logprob": -0.39418065547943115, "total_sequence_tokens": 41 }, { "index": 1, "tokens": 3, "sum_logprob": -1.770630121231079, "mean_logprob": -0.5902100404103597, "total_sequence_tokens": 41 }, { "index": 2, "tokens": 2, "sum_logprob": -9.81666088104248, "mean_logprob": -4.90833044052124, "total_sequence_tokens": 40 }, { "index": 3, "tokens": 2, "sum_logprob": -13.6995849609375, "mean_logprob": -6.84979248046875, "total_sequence_tokens": 40 } ], "best_vs_second_margin": 0.19602938493092859, "context_truncated": false, "elapsed_seconds": 0.03791658200134407 }, { "id": "context_tracking-050", "category": "context_tracking", "difficulty": "hard", "category_weight": 1.2, "difficulty_weight": 2.25, "point_weight": 2.6999999999999997, "earned_points": 0.0, "item_elo": 1100, "context": "The sequence begins amber, cobalt, jade, silver. Remove cobalt, place silver first, then insert rose immediately after amber. The final sequence is", "continuations": [ " silver, rose, amber, jade.", " silver, amber, rose, jade.", " amber, rose, silver, jade.", " silver, amber, jade, rose." ], "tags": [ "permutation" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 10, "sum_logprob": -9.478717803955078, "mean_logprob": -0.9478717803955078, "total_sequence_tokens": 46 }, { "index": 1, "tokens": 10, "sum_logprob": -9.959565162658691, "mean_logprob": -0.9959565162658691, "total_sequence_tokens": 46 }, { "index": 2, "tokens": 10, "sum_logprob": -11.633020401000977, "mean_logprob": -1.1633020401000977, "total_sequence_tokens": 46 }, { "index": 3, "tokens": 10, "sum_logprob": -15.396784782409668, "mean_logprob": -1.5396784782409667, "total_sequence_tokens": 46 } ], "best_vs_second_margin": 0.04808473587036133, "context_truncated": false, "elapsed_seconds": 0.044577638000191655 }, { "id": "quantitative-001", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "Seven plus five equals", "continuations": [ " twelve.", " eleven.", " thirteen.", " fourteen." ], "tags": [ "arithmetic" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -5.292641639709473, "mean_logprob": -2.6463208198547363, "total_sequence_tokens": 7 }, { "index": 1, "tokens": 3, "sum_logprob": -5.131879806518555, "mean_logprob": -1.7106266021728516, "total_sequence_tokens": 8 }, { "index": 2, "tokens": 3, "sum_logprob": -6.535145282745361, "mean_logprob": -2.1783817609151206, "total_sequence_tokens": 8 }, { "index": 3, "tokens": 3, "sum_logprob": -7.454742431640625, "mean_logprob": -2.4849141438802085, "total_sequence_tokens": 8 } ], "best_vs_second_margin": 0.46775515874226903, "context_truncated": false, "elapsed_seconds": 0.033248167999772704 }, { "id": "quantitative-002", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "A box holds nine pencils. After three are removed, it holds", "continuations": [ " five pencils.", " twelve pencils.", " seven pencils.", " six pencils." ], "tags": [ "subtraction" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -3.5479001998901367, "mean_logprob": -1.182633399963379, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 3, "sum_logprob": -3.5878844261169434, "mean_logprob": -1.1959614753723145, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 3, "sum_logprob": -4.939874649047852, "mean_logprob": -1.6466248830159504, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 3, "sum_logprob": -4.488526344299316, "mean_logprob": -1.4961754480997722, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 0.013328075408935547, "context_truncated": false, "elapsed_seconds": 0.02531888900011836 }, { "id": "quantitative-003", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "Four groups of three objects contain", "continuations": [ " seven objects.", " ten objects.", " twelve objects.", " sixteen objects." ], "tags": [ "multiplication" ], "prediction": 3, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -11.508077621459961, "mean_logprob": -3.836025873819987, "total_sequence_tokens": 9 }, { "index": 1, "tokens": 3, "sum_logprob": -10.469986915588379, "mean_logprob": -3.4899956385294595, "total_sequence_tokens": 9 }, { "index": 2, "tokens": 3, "sum_logprob": -13.069352149963379, "mean_logprob": -4.35645071665446, "total_sequence_tokens": 9 }, { "index": 3, "tokens": 4, "sum_logprob": -13.693161964416504, "mean_logprob": -3.423290491104126, "total_sequence_tokens": 10 } ], "best_vs_second_margin": 0.06670514742533351, "context_truncated": false, "elapsed_seconds": 0.028185780000058003 }, { "id": "quantitative-004", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 1.3, "item_elo": 900, "context": "Twenty divided equally among five people gives each person", "continuations": [ " two.", " three.", " five.", " four." ], "tags": [ "division" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -8.99081802368164, "mean_logprob": -4.49540901184082, "total_sequence_tokens": 12 }, { "index": 1, "tokens": 2, "sum_logprob": -8.719918251037598, "mean_logprob": -4.359959125518799, "total_sequence_tokens": 12 }, { "index": 2, "tokens": 2, "sum_logprob": -9.545299530029297, "mean_logprob": -4.772649765014648, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 2, "sum_logprob": -7.664897918701172, "mean_logprob": -3.832448959350586, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 0.5275101661682129, "context_truncated": false, "elapsed_seconds": 0.025594724000256974 }, { "id": "quantitative-005", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 1.3, "item_elo": 900, "context": "The number immediately after 199 is", "continuations": [ " 200.", " 198.", " 209.", " 299." ], "tags": [ "number_sense" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -2.5621232986450195, "mean_logprob": -0.5124246597290039, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 5, "sum_logprob": -5.487677574157715, "mean_logprob": -1.0975355148315429, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 5, "sum_logprob": -8.877996444702148, "mean_logprob": -1.7755992889404297, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 5, "sum_logprob": -6.280266761779785, "mean_logprob": -1.256053352355957, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 0.585110855102539, "context_truncated": false, "elapsed_seconds": 0.024640380001073936 }, { "id": "quantitative-006", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "Half of eighteen is", "continuations": [ " eight.", " nine.", " ten.", " thirty-six." ], "tags": [ "fractions" ], "prediction": 3, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -5.813625812530518, "mean_logprob": -2.906812906265259, "total_sequence_tokens": 7 }, { "index": 1, "tokens": 2, "sum_logprob": -6.29595947265625, "mean_logprob": -3.147979736328125, "total_sequence_tokens": 7 }, { "index": 2, "tokens": 2, "sum_logprob": -6.812882423400879, "mean_logprob": -3.4064412117004395, "total_sequence_tokens": 7 }, { "index": 3, "tokens": 4, "sum_logprob": -8.896859169006348, "mean_logprob": -2.224214792251587, "total_sequence_tokens": 9 } ], "best_vs_second_margin": 0.6825981140136719, "context_truncated": false, "elapsed_seconds": 0.02822259400090843 }, { "id": "quantitative-007", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "A rectangle with length five and width two has area", "continuations": [ " seven.", " twelve.", " ten.", " fourteen." ], "tags": [ "geometry" ], "prediction": 3, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -6.672070503234863, "mean_logprob": -3.3360352516174316, "total_sequence_tokens": 12 }, { "index": 1, "tokens": 2, "sum_logprob": -8.611666679382324, "mean_logprob": -4.305833339691162, "total_sequence_tokens": 12 }, { "index": 2, "tokens": 2, "sum_logprob": -7.9228715896606445, "mean_logprob": -3.9614357948303223, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 3, "sum_logprob": -7.710935592651367, "mean_logprob": -2.5703118642171225, "total_sequence_tokens": 13 } ], "best_vs_second_margin": 0.7657233874003091, "context_truncated": false, "elapsed_seconds": 0.02752333599892154 }, { "id": "quantitative-008", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 1.3, "item_elo": 900, "context": "Three quarters of an hour is", "continuations": [ " thirty minutes.", " forty minutes.", " sixty minutes.", " forty-five minutes." ], "tags": [ "time" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -8.348872184753418, "mean_logprob": -2.782957394917806, "total_sequence_tokens": 9 }, { "index": 1, "tokens": 3, "sum_logprob": -9.526144027709961, "mean_logprob": -3.175381342569987, "total_sequence_tokens": 9 }, { "index": 2, "tokens": 5, "sum_logprob": -10.311846733093262, "mean_logprob": -2.0623693466186523, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 5, "sum_logprob": -9.686224937438965, "mean_logprob": -1.937244987487793, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 0.1251243591308593, "context_truncated": false, "elapsed_seconds": 0.03023112299888453 }, { "id": "quantitative-009", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "A ten-euro item discounted by two euros costs", "continuations": [ " eight euros.", " twelve euros.", " five euros.", " twenty euros." ], "tags": [ "money" ], "prediction": 2, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -10.715323448181152, "mean_logprob": -2.1430646896362306, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 5, "sum_logprob": -13.067676544189453, "mean_logprob": -2.6135353088378905, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 5, "sum_logprob": -9.866440773010254, "mean_logprob": -1.9732881546020509, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 5, "sum_logprob": -11.636058807373047, "mean_logprob": -2.3272117614746093, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.16977653503417978, "context_truncated": false, "elapsed_seconds": 0.02641895899978408 }, { "id": "quantitative-010", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 1.3, "item_elo": 900, "context": "The perimeter of a square with side length three is", "continuations": [ " nine.", " twelve.", " six.", " fifteen." ], "tags": [ "geometry" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -11.847151756286621, "mean_logprob": -5.9235758781433105, "total_sequence_tokens": 12 }, { "index": 1, "tokens": 2, "sum_logprob": -11.464617729187012, "mean_logprob": -5.732308864593506, "total_sequence_tokens": 12 }, { "index": 2, "tokens": 2, "sum_logprob": -11.502902030944824, "mean_logprob": -5.751451015472412, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 2, "sum_logprob": -11.837128639221191, "mean_logprob": -5.918564319610596, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 0.01914215087890625, "context_truncated": false, "elapsed_seconds": 0.02509022899903357 }, { "id": "quantitative-011", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 1.3, "item_elo": 900, "context": "Thirty is five more than", "continuations": [ " twenty.", " thirty-five.", " twenty-five.", " fifteen." ], "tags": [ "arithmetic" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -4.370925426483154, "mean_logprob": -2.185462713241577, "total_sequence_tokens": 9 }, { "index": 1, "tokens": 4, "sum_logprob": -8.961140632629395, "mean_logprob": -2.2402851581573486, "total_sequence_tokens": 11 }, { "index": 2, "tokens": 4, "sum_logprob": -6.706611156463623, "mean_logprob": -1.6766527891159058, "total_sequence_tokens": 11 }, { "index": 3, "tokens": 2, "sum_logprob": -6.271145820617676, "mean_logprob": -3.135572910308838, "total_sequence_tokens": 9 } ], "best_vs_second_margin": 0.5088099241256714, "context_truncated": false, "elapsed_seconds": 0.027746277999540325 }, { "id": "quantitative-012", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 1.3, "item_elo": 900, "context": "If one notebook costs four euros, two notebooks cost", "continuations": [ " four euros.", " six euros.", " twelve euros.", " eight euros." ], "tags": [ "money" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -5.146291255950928, "mean_logprob": -1.0292582511901855, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 5, "sum_logprob": -5.293211936950684, "mean_logprob": -1.0586423873901367, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 5, "sum_logprob": -7.491818904876709, "mean_logprob": -1.4983637809753418, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 5, "sum_logprob": -4.346880912780762, "mean_logprob": -0.8693761825561523, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 0.1598820686340332, "context_truncated": false, "elapsed_seconds": 0.026342607001424767 }, { "id": "quantitative-013", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "The difference between fifteen and nine is", "continuations": [ " six.", " five.", " seven.", " twenty-four." ], "tags": [ "subtraction" ], "prediction": 2, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -5.347989082336426, "mean_logprob": -2.673994541168213, "total_sequence_tokens": 9 }, { "index": 1, "tokens": 2, "sum_logprob": -4.929468631744385, "mean_logprob": -2.4647343158721924, "total_sequence_tokens": 9 }, { "index": 2, "tokens": 2, "sum_logprob": -4.034341812133789, "mean_logprob": -2.0171709060668945, "total_sequence_tokens": 9 }, { "index": 3, "tokens": 4, "sum_logprob": -9.035197257995605, "mean_logprob": -2.2587993144989014, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 0.24162840843200684, "context_truncated": false, "elapsed_seconds": 0.025635102998421644 }, { "id": "quantitative-014", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "Counting by tens, the number after forty is", "continuations": [ " forty-one.", " fifty.", " sixty.", " thirty." ], "tags": [ "number_sense" ], "prediction": 3, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -6.080322265625, "mean_logprob": -1.52008056640625, "total_sequence_tokens": 14 }, { "index": 1, "tokens": 2, "sum_logprob": -4.013950347900391, "mean_logprob": -2.0069751739501953, "total_sequence_tokens": 12 }, { "index": 2, "tokens": 4, "sum_logprob": -7.232993125915527, "mean_logprob": -1.8082482814788818, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 2, "sum_logprob": -2.881405830383301, "mean_logprob": -1.4407029151916504, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 0.07937765121459961, "context_truncated": false, "elapsed_seconds": 0.025576610998541582 }, { "id": "quantitative-015", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "Two meters equal", "continuations": [ " twenty centimeters.", " two hundred millimeters.", " two hundred centimeters.", " two thousand centimeters." ], "tags": [ "units" ], "prediction": 1, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -14.974717140197754, "mean_logprob": -4.991572380065918, "total_sequence_tokens": 6 }, { "index": 1, "tokens": 5, "sum_logprob": -16.25657081604004, "mean_logprob": -3.251314163208008, "total_sequence_tokens": 8 }, { "index": 2, "tokens": 4, "sum_logprob": -15.64132308959961, "mean_logprob": -3.9103307723999023, "total_sequence_tokens": 7 }, { "index": 3, "tokens": 4, "sum_logprob": -17.949125289916992, "mean_logprob": -4.487281322479248, "total_sequence_tokens": 7 } ], "best_vs_second_margin": 0.6590166091918945, "context_truncated": false, "elapsed_seconds": 0.027211262000491843 }, { "id": "quantitative-016", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "The average of four and eight is", "continuations": [ " four.", " five.", " eight.", " six." ], "tags": [ "average" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -5.802121162414551, "mean_logprob": -2.9010605812072754, "total_sequence_tokens": 9 }, { "index": 1, "tokens": 2, "sum_logprob": -6.14479398727417, "mean_logprob": -3.072396993637085, "total_sequence_tokens": 9 }, { "index": 2, "tokens": 2, "sum_logprob": -6.672871112823486, "mean_logprob": -3.336435556411743, "total_sequence_tokens": 9 }, { "index": 3, "tokens": 2, "sum_logprob": -6.14555025100708, "mean_logprob": -3.07277512550354, "total_sequence_tokens": 9 } ], "best_vs_second_margin": 0.17133641242980957, "context_truncated": false, "elapsed_seconds": 0.02484983700014709 }, { "id": "quantitative-017", "category": "quantitative", "difficulty": "easy", "category_weight": 1.3, "difficulty_weight": 1.0, "point_weight": 1.3, "earned_points": 0.0, "item_elo": 900, "context": "A dozen eggs contains", "continuations": [ " twelve eggs.", " ten eggs.", " six eggs.", " twenty eggs." ], "tags": [ "number_sense" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -9.102995872497559, "mean_logprob": -3.034331957499186, "total_sequence_tokens": 7 }, { "index": 1, "tokens": 3, "sum_logprob": -7.773576259613037, "mean_logprob": -2.591192086537679, "total_sequence_tokens": 7 }, { "index": 2, "tokens": 3, "sum_logprob": -8.962491035461426, "mean_logprob": -2.987497011820475, "total_sequence_tokens": 7 }, { "index": 3, "tokens": 3, "sum_logprob": -8.10993480682373, "mean_logprob": -2.7033116022745767, "total_sequence_tokens": 7 } ], "best_vs_second_margin": 0.11211951573689749, "context_truncated": false, "elapsed_seconds": 0.024494163000781555 }, { "id": "quantitative-018", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "A train travels 180 kilometers in three hours at a constant rate. Its speed is", "continuations": [ " forty kilometers per hour.", " sixty kilometers per hour.", " ninety kilometers per hour.", " one hundred eighty kilometers per hour." ], "tags": [ "rate" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -16.961780548095703, "mean_logprob": -3.392356109619141, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 7, "sum_logprob": -26.406064987182617, "mean_logprob": -3.7722949981689453, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 7, "sum_logprob": -17.92763900756836, "mean_logprob": -2.56109128679548, "total_sequence_tokens": 26 }, { "index": 3, "tokens": 8, "sum_logprob": -23.113128662109375, "mean_logprob": -2.889141082763672, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.32804979596819184, "context_truncated": false, "elapsed_seconds": 0.029687533000469557 }, { "id": "quantitative-019", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "A jacket priced at 80 euros is reduced by 25 percent. The sale price is", "continuations": [ " fifty euros.", " fifty-five euros.", " sixty euros.", " sixty-five euros." ], "tags": [ "percent" ], "prediction": 3, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -18.02532386779785, "mean_logprob": -3.6050647735595702, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 7, "sum_logprob": -19.415939331054688, "mean_logprob": -2.773705618722098, "total_sequence_tokens": 31 }, { "index": 2, "tokens": 7, "sum_logprob": -17.058008193969727, "mean_logprob": -2.4368583134242465, "total_sequence_tokens": 31 }, { "index": 3, "tokens": 9, "sum_logprob": -13.648637771606445, "mean_logprob": -1.5165153079562717, "total_sequence_tokens": 33 } ], "best_vs_second_margin": 0.9203430054679749, "context_truncated": false, "elapsed_seconds": 0.04079642699980468 }, { "id": "quantitative-020", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 1.9500000000000002, "item_elo": 1000, "context": "A tank contains 36 liters. One third is drained, leaving", "continuations": [ " twelve liters.", " eighteen liters.", " thirty liters.", " twenty-four liters." ], "tags": [ "fractions" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -14.342558860778809, "mean_logprob": -4.7808529535929365, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 4, "sum_logprob": -16.174556732177734, "mean_logprob": -4.043639183044434, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 3, "sum_logprob": -16.89756202697754, "mean_logprob": -5.63252067565918, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 5, "sum_logprob": -15.412789344787598, "mean_logprob": -3.0825578689575197, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.9610813140869139, "context_truncated": false, "elapsed_seconds": 0.045352904999163 }, { "id": "quantitative-021", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "Five consecutive integers begin with 12. Their sum is", "continuations": [ " seventy.", " sixty.", " seventy-five.", " eighty." ], "tags": [ "sequence" ], "prediction": 2, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -17.021854400634766, "mean_logprob": -5.673951466878255, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 4, "sum_logprob": -23.48359489440918, "mean_logprob": -5.870898723602295, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 5, "sum_logprob": -14.351556777954102, "mean_logprob": -2.8703113555908204, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 3, "sum_logprob": -17.526702880859375, "mean_logprob": -5.842234293619792, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 2.8036401112874345, "context_truncated": false, "elapsed_seconds": 0.031220798000504146 }, { "id": "quantitative-022", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "A right triangle has legs of length 6 and 8. Its hypotenuse has length", "continuations": [ " nine.", " ten.", " twelve.", " fourteen." ], "tags": [ "geometry" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -11.915306091308594, "mean_logprob": -5.957653045654297, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 2, "sum_logprob": -13.793683052062988, "mean_logprob": -6.896841526031494, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 2, "sum_logprob": -13.201858520507812, "mean_logprob": -6.600929260253906, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 3, "sum_logprob": -18.208065032958984, "mean_logprob": -6.069355010986328, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.11170196533203125, "context_truncated": false, "elapsed_seconds": 0.027931208000154584 }, { "id": "quantitative-023", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "If x plus 7 equals 19, then x equals", "continuations": [ " ten.", " eleven.", " twelve.", " twenty-six." ], "tags": [ "algebra" ], "prediction": 3, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -11.407376289367676, "mean_logprob": -5.703688144683838, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 3, "sum_logprob": -13.684794425964355, "mean_logprob": -4.561598141988118, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 2, "sum_logprob": -13.927817344665527, "mean_logprob": -6.963908672332764, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 4, "sum_logprob": -15.525569915771484, "mean_logprob": -3.881392478942871, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 0.6802056630452471, "context_truncated": false, "elapsed_seconds": 0.03209438700105238 }, { "id": "quantitative-024", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 1.9500000000000002, "item_elo": 1000, "context": "A worker earns 18 euros per hour for seven hours. Total earnings are", "continuations": [ " 108 euros.", " 116 euros.", " 136 euros.", " 126 euros." ], "tags": [ "multiplication" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -9.774751663208008, "mean_logprob": -1.221843957901001, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 8, "sum_logprob": -10.015142440795898, "mean_logprob": -1.2518928050994873, "total_sequence_tokens": 27 }, { "index": 2, "tokens": 8, "sum_logprob": -11.240349769592285, "mean_logprob": -1.4050437211990356, "total_sequence_tokens": 27 }, { "index": 3, "tokens": 8, "sum_logprob": -8.623146057128906, "mean_logprob": -1.0778932571411133, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.1439507007598877, "context_truncated": false, "elapsed_seconds": 0.06568012900061149 }, { "id": "quantitative-025", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 1.9500000000000002, "item_elo": 1000, "context": "The ratio of red to blue marbles is 3 to 5. If there are 15 red marbles, there are", "continuations": [ " twenty-five blue marbles.", " nine blue marbles.", " eighteen blue marbles.", " seventy-five blue marbles." ], "tags": [ "ratio" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -16.056371688842773, "mean_logprob": -2.6760619481404624, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 4, "sum_logprob": -12.535652160644531, "mean_logprob": -3.133913040161133, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 5, "sum_logprob": -17.886301040649414, "mean_logprob": -3.577260208129883, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 7, "sum_logprob": -20.623779296875, "mean_logprob": -2.946254185267857, "total_sequence_tokens": 32 } ], "best_vs_second_margin": 0.2701922371273948, "context_truncated": false, "elapsed_seconds": 0.04084540600160835 }, { "id": "quantitative-026", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "A circle has radius 4. Using pi as 3.14, its circumference is approximately", "continuations": [ " 12.56.", " 25.12.", " 50.24.", " 16.00." ], "tags": [ "geometry" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -3.247330665588379, "mean_logprob": -0.46390438079833984, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 7, "sum_logprob": -11.988222122192383, "mean_logprob": -1.7126031603131975, "total_sequence_tokens": 27 }, { "index": 2, "tokens": 7, "sum_logprob": -10.607452392578125, "mean_logprob": -1.515350341796875, "total_sequence_tokens": 27 }, { "index": 3, "tokens": 7, "sum_logprob": -26.597305297851562, "mean_logprob": -3.799615042550223, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 1.0514459609985352, "context_truncated": false, "elapsed_seconds": 0.026777819999551866 }, { "id": "quantitative-027", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 1.9500000000000002, "item_elo": 1000, "context": "An account balance rises from 240 to 300. The percentage increase is", "continuations": [ " fifteen percent.", " twenty percent.", " twenty-five percent.", " sixty percent." ], "tags": [ "percent" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -19.058584213256836, "mean_logprob": -6.352861404418945, "total_sequence_tokens": 22 }, { "index": 1, "tokens": 3, "sum_logprob": -19.500022888183594, "mean_logprob": -6.500007629394531, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 5, "sum_logprob": -18.61857795715332, "mean_logprob": -3.723715591430664, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 5, "sum_logprob": -19.820444107055664, "mean_logprob": -3.9640888214111327, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 0.24037322998046884, "context_truncated": false, "elapsed_seconds": 0.02628384899981029 }, { "id": "quantitative-028", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "A bus leaves at 09:35 and arrives at 11:05. The journey lasts", "continuations": [ " seventy minutes.", " eighty minutes.", " one hundred minutes.", " ninety minutes." ], "tags": [ "time" ], "prediction": 2, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -19.287830352783203, "mean_logprob": -4.821957588195801, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 4, "sum_logprob": -33.98118591308594, "mean_logprob": -8.495296478271484, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 4, "sum_logprob": -19.279085159301758, "mean_logprob": -4.8197712898254395, "total_sequence_tokens": 29 }, { "index": 3, "tokens": 5, "sum_logprob": -30.71573829650879, "mean_logprob": -6.143147659301758, "total_sequence_tokens": 30 } ], "best_vs_second_margin": 0.002186298370361328, "context_truncated": false, "elapsed_seconds": 0.02756844000032288 }, { "id": "quantitative-029", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 1.9500000000000002, "item_elo": 1000, "context": "The median of 3, 7, 8, 12, and 20 is", "continuations": [ " eight.", " seven.", " ten.", " twelve." ], "tags": [ "statistics" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -12.059478759765625, "mean_logprob": -6.0297393798828125, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 2, "sum_logprob": -12.399323463439941, "mean_logprob": -6.199661731719971, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 2, "sum_logprob": -14.319886207580566, "mean_logprob": -7.159943103790283, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 2, "sum_logprob": -14.025663375854492, "mean_logprob": -7.012831687927246, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.1699223518371582, "context_truncated": false, "elapsed_seconds": 0.05327712399957818 }, { "id": "quantitative-030", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "A 48-page booklet is printed with four pages per sheet. It requires", "continuations": [ " ten sheets.", " twelve sheets.", " sixteen sheets.", " forty-four sheets." ], "tags": [ "division" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -11.124528884887695, "mean_logprob": -3.708176294962565, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 3, "sum_logprob": -10.980240821838379, "mean_logprob": -3.6600802739461265, "total_sequence_tokens": 21 }, { "index": 2, "tokens": 4, "sum_logprob": -12.329124450683594, "mean_logprob": -3.0822811126708984, "total_sequence_tokens": 22 }, { "index": 3, "tokens": 5, "sum_logprob": -16.06705093383789, "mean_logprob": -3.213410186767578, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.1311290740966795, "context_truncated": false, "elapsed_seconds": 0.05290328900082386 }, { "id": "quantitative-031", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "Three fifths written as a decimal is", "continuations": [ " 0.3.", " 0.5.", " 0.6.", " 0.8." ], "tags": [ "fractions" ], "prediction": 3, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -7.394290924072266, "mean_logprob": -1.478858184814453, "total_sequence_tokens": 13 }, { "index": 1, "tokens": 5, "sum_logprob": -4.609033107757568, "mean_logprob": -0.9218066215515137, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 5, "sum_logprob": -5.725163459777832, "mean_logprob": -1.1450326919555665, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 5, "sum_logprob": -4.468921661376953, "mean_logprob": -0.8937843322753907, "total_sequence_tokens": 13 } ], "best_vs_second_margin": 0.028022289276123047, "context_truncated": false, "elapsed_seconds": 0.025549125999532407 }, { "id": "quantitative-032", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 1.9500000000000002, "item_elo": 1000, "context": "A shop buys an item for 30 euros and sells it for 42 euros. The profit is", "continuations": [ " 10 euros.", " 11 euros.", " 72 euros.", " 12 euros." ], "tags": [ "money" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -3.9913277626037598, "mean_logprob": -0.5701896803719657, "total_sequence_tokens": 33 }, { "index": 1, "tokens": 7, "sum_logprob": -6.558568954467773, "mean_logprob": -0.9369384220668248, "total_sequence_tokens": 33 }, { "index": 2, "tokens": 7, "sum_logprob": -8.563762664794922, "mean_logprob": -1.2233946663992745, "total_sequence_tokens": 33 }, { "index": 3, "tokens": 7, "sum_logprob": -3.681147336959839, "mean_logprob": -0.5258781909942627, "total_sequence_tokens": 33 } ], "best_vs_second_margin": 0.04431148937770302, "context_truncated": false, "elapsed_seconds": 0.03577348799990432 }, { "id": "quantitative-033", "category": "quantitative", "difficulty": "medium", "category_weight": 1.3, "difficulty_weight": 1.5, "point_weight": 1.9500000000000002, "earned_points": 0.0, "item_elo": 1000, "context": "If 2x minus 4 equals 16, then x equals", "continuations": [ " ten.", " six.", " eight.", " twelve." ], "tags": [ "algebra" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -10.682738304138184, "mean_logprob": -5.341369152069092, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 2, "sum_logprob": -10.106553077697754, "mean_logprob": -5.053276538848877, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 2, "sum_logprob": -10.52470874786377, "mean_logprob": -5.262354373931885, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 2, "sum_logprob": -10.924354553222656, "mean_logprob": -5.462177276611328, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 0.2090778350830078, "context_truncated": false, "elapsed_seconds": 0.03075260699915816 }, { "id": "quantitative-034", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A price increases by 20 percent and is then reduced by 20 percent. Starting from 100 euros, the final price is", "continuations": [ " 100 euros.", " 96 euros.", " 80 euros.", " 104 euros." ], "tags": [ "percent" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -14.256305694580078, "mean_logprob": -1.7820382118225098, "total_sequence_tokens": 40 }, { "index": 1, "tokens": 7, "sum_logprob": -10.194211959838867, "mean_logprob": -1.4563159942626953, "total_sequence_tokens": 39 }, { "index": 2, "tokens": 7, "sum_logprob": -7.444924831390381, "mean_logprob": -1.0635606901986259, "total_sequence_tokens": 39 }, { "index": 3, "tokens": 8, "sum_logprob": -11.327791213989258, "mean_logprob": -1.4159739017486572, "total_sequence_tokens": 40 } ], "best_vs_second_margin": 0.35241321155003136, "context_truncated": false, "elapsed_seconds": 0.0364853069986566 }, { "id": "quantitative-035", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 2.9250000000000003, "item_elo": 1150, "context": "A mixture contains milk and water in a 5 to 2 ratio. In 21 liters of mixture, the amount of water is", "continuations": [ " five liters.", " seven liters.", " six liters.", " fifteen liters." ], "tags": [ "ratio" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -14.469293594360352, "mean_logprob": -4.823097864786784, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 3, "sum_logprob": -16.154003143310547, "mean_logprob": -5.384667714436849, "total_sequence_tokens": 31 }, { "index": 2, "tokens": 3, "sum_logprob": -13.7432279586792, "mean_logprob": -4.581075986226399, "total_sequence_tokens": 31 }, { "index": 3, "tokens": 3, "sum_logprob": -15.343059539794922, "mean_logprob": -5.114353179931641, "total_sequence_tokens": 31 } ], "best_vs_second_margin": 0.2420218785603847, "context_truncated": false, "elapsed_seconds": 0.036742966000019806 }, { "id": "quantitative-036", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 2.9250000000000003, "item_elo": 1150, "context": "Two numbers have sum 46 and difference 8. The larger number is", "continuations": [ " nineteen.", " twenty-three.", " thirty-five.", " twenty-seven." ], "tags": [ "algebra" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -20.145109176635742, "mean_logprob": -5.0362772941589355, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 4, "sum_logprob": -15.645545959472656, "mean_logprob": -3.911386489868164, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 4, "sum_logprob": -14.99804973602295, "mean_logprob": -3.7495124340057373, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 4, "sum_logprob": -14.937511444091797, "mean_logprob": -3.734377861022949, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.015134572982788086, "context_truncated": false, "elapsed_seconds": 0.027836683999339584 }, { "id": "quantitative-037", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A car travels the first 120 kilometers at 60 km/h and the next 120 at 40 km/h. Its average speed for the full trip is", "continuations": [ " 48 km/h.", " 50 km/h.", " 45 km/h.", " 52 km/h." ], "tags": [ "rate" ], "prediction": 2, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -6.520872116088867, "mean_logprob": -0.9315531594412667, "total_sequence_tokens": 47 }, { "index": 1, "tokens": 7, "sum_logprob": -3.3077588081359863, "mean_logprob": -0.4725369725908552, "total_sequence_tokens": 47 }, { "index": 2, "tokens": 7, "sum_logprob": -0.6295626163482666, "mean_logprob": -0.08993751662118095, "total_sequence_tokens": 47 }, { "index": 3, "tokens": 7, "sum_logprob": -4.594418048858643, "mean_logprob": -0.6563454355512347, "total_sequence_tokens": 47 } ], "best_vs_second_margin": 0.38259945596967426, "context_truncated": false, "elapsed_seconds": 0.03457150299982459 }, { "id": "quantitative-038", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A rectangle's length is three more than its width. If its perimeter is 30, its width is", "continuations": [ " five.", " six.", " seven.", " nine." ], "tags": [ "algebra" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -11.429022789001465, "mean_logprob": -5.714511394500732, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 2, "sum_logprob": -11.57111930847168, "mean_logprob": -5.78555965423584, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 2, "sum_logprob": -12.157730102539062, "mean_logprob": -6.078865051269531, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 2, "sum_logprob": -13.113155364990234, "mean_logprob": -6.556577682495117, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.07104825973510742, "context_truncated": false, "elapsed_seconds": 0.031772250000358326 }, { "id": "quantitative-039", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A bag has four red, three blue, and five green balls. The probability of drawing a blue ball is", "continuations": [ " one third.", " three fifths.", " one quarter.", " three quarters." ], "tags": [ "probability" ], "prediction": 1, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -17.496288299560547, "mean_logprob": -5.832096099853516, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 4, "sum_logprob": -15.568428039550781, "mean_logprob": -3.8921070098876953, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 3, "sum_logprob": -16.224609375, "mean_logprob": -5.408203125, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 3, "sum_logprob": -17.390769958496094, "mean_logprob": -5.796923319498698, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 1.5160961151123047, "context_truncated": false, "elapsed_seconds": 0.03168500199899427 }, { "id": "quantitative-040", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "An investment of 2,000 euros earns simple interest at 4 percent per year for three years. The interest earned is", "continuations": [ " 80 euros.", " 160 euros.", " 2,240 euros.", " 240 euros." ], "tags": [ "finance" ], "prediction": 1, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -8.198927879333496, "mean_logprob": -1.1712754113333566, "total_sequence_tokens": 37 }, { "index": 1, "tokens": 8, "sum_logprob": -8.330878257751465, "mean_logprob": -1.041359782218933, "total_sequence_tokens": 38 }, { "index": 2, "tokens": 10, "sum_logprob": -17.7349910736084, "mean_logprob": -1.77349910736084, "total_sequence_tokens": 40 }, { "index": 3, "tokens": 8, "sum_logprob": -8.385602951049805, "mean_logprob": -1.0482003688812256, "total_sequence_tokens": 38 } ], "best_vs_second_margin": 0.0068405866622924805, "context_truncated": false, "elapsed_seconds": 0.036835486000200035 }, { "id": "quantitative-041", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "The sequence 2, 6, 12, 20, 30 continues with", "continuations": [ " 42.", " 36.", " 40.", " 44." ], "tags": [ "sequence" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -12.798002243041992, "mean_logprob": -3.199500560760498, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 4, "sum_logprob": -11.089204788208008, "mean_logprob": -2.772301197052002, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 4, "sum_logprob": -12.132857322692871, "mean_logprob": -3.0332143306732178, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 4, "sum_logprob": -12.851228713989258, "mean_logprob": -3.2128071784973145, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.2609131336212158, "context_truncated": false, "elapsed_seconds": 0.030952609000451048 }, { "id": "quantitative-042", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A machine completes three jobs in 18 minutes at a constant rate. At that rate, eight jobs take", "continuations": [ " 36 minutes.", " 48 minutes.", " 54 minutes.", " 144 minutes." ], "tags": [ "rate" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -4.16306209564209, "mean_logprob": -0.832612419128418, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 5, "sum_logprob": -4.029392242431641, "mean_logprob": -0.8058784484863282, "total_sequence_tokens": 27 }, { "index": 2, "tokens": 5, "sum_logprob": -3.299407720565796, "mean_logprob": -0.6598815441131591, "total_sequence_tokens": 27 }, { "index": 3, "tokens": 6, "sum_logprob": -7.349918842315674, "mean_logprob": -1.224986473719279, "total_sequence_tokens": 28 } ], "best_vs_second_margin": 0.14599690437316903, "context_truncated": false, "elapsed_seconds": 0.027485315999001614 }, { "id": "quantitative-043", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 2.9250000000000003, "item_elo": 1150, "context": "The mean of six numbers is 14. Five of them sum to 61. The sixth number is", "continuations": [ " fourteen.", " seventeen.", " twenty-three.", " eighty-four." ], "tags": [ "statistics" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -16.144824981689453, "mean_logprob": -5.381608327229817, "total_sequence_tokens": 26 }, { "index": 1, "tokens": 3, "sum_logprob": -11.930179595947266, "mean_logprob": -3.976726531982422, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 4, "sum_logprob": -13.954082489013672, "mean_logprob": -3.488520622253418, "total_sequence_tokens": 27 }, { "index": 3, "tokens": 5, "sum_logprob": -19.000680923461914, "mean_logprob": -3.800136184692383, "total_sequence_tokens": 28 } ], "best_vs_second_margin": 0.31161556243896493, "context_truncated": false, "elapsed_seconds": 0.025930818999768235 }, { "id": "quantitative-044", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A map scale is 1 to 50,000. A distance of 6 centimeters on the map represents", "continuations": [ " 300 meters.", " 30 kilometers.", " 300 kilometers.", " 3 kilometers." ], "tags": [ "scale" ], "prediction": 2, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -9.360454559326172, "mean_logprob": -1.5600757598876953, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 5, "sum_logprob": -9.09926986694336, "mean_logprob": -1.8198539733886718, "total_sequence_tokens": 30 }, { "index": 2, "tokens": 6, "sum_logprob": -8.061050415039062, "mean_logprob": -1.3435084025065105, "total_sequence_tokens": 31 }, { "index": 3, "tokens": 4, "sum_logprob": -7.9915056228637695, "mean_logprob": -1.9978764057159424, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.21656735738118482, "context_truncated": false, "elapsed_seconds": 0.0256035399997927 }, { "id": "quantitative-045", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 2.9250000000000003, "item_elo": 1150, "context": "If three quarters of a number is 27, the number is", "continuations": [ " thirty-six.", " twenty-one.", " thirty.", " forty." ], "tags": [ "algebra" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -13.211942672729492, "mean_logprob": -3.302985668182373, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 4, "sum_logprob": -15.152857780456543, "mean_logprob": -3.7882144451141357, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 2, "sum_logprob": -15.172837257385254, "mean_logprob": -7.586418628692627, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 2, "sum_logprob": -13.899062156677246, "mean_logprob": -6.949531078338623, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 0.4852287769317627, "context_truncated": false, "elapsed_seconds": 0.02752245900046546 }, { "id": "quantitative-046", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A 15-liter solution is 40 percent salt. Adding 5 liters of water makes the new salt concentration", "continuations": [ " twenty percent.", " thirty percent.", " thirty-five percent.", " forty-five percent." ], "tags": [ "mixture" ], "prediction": 3, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -20.30893325805664, "mean_logprob": -6.769644419352214, "total_sequence_tokens": 28 }, { "index": 1, "tokens": 3, "sum_logprob": -20.39630126953125, "mean_logprob": -6.79876708984375, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 5, "sum_logprob": -17.16763687133789, "mean_logprob": -3.433527374267578, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 5, "sum_logprob": -16.944379806518555, "mean_logprob": -3.388875961303711, "total_sequence_tokens": 30 } ], "best_vs_second_margin": 0.0446514129638671, "context_truncated": false, "elapsed_seconds": 0.029969453000376234 }, { "id": "quantitative-047", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A square's diagonal has length 10. Its area is", "continuations": [ " twenty-five.", " forty.", " fifty.", " one hundred." ], "tags": [ "geometry" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -17.12101936340332, "mean_logprob": -4.28025484085083, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 2, "sum_logprob": -21.154712677001953, "mean_logprob": -10.577356338500977, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 2, "sum_logprob": -21.690587997436523, "mean_logprob": -10.845293998718262, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 3, "sum_logprob": -22.510478973388672, "mean_logprob": -7.503492991129558, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 3.2232381502787275, "context_truncated": false, "elapsed_seconds": 0.03216763900127262 }, { "id": "quantitative-048", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A clock gains two minutes every three hours. After 24 real hours, it will be ahead by", "continuations": [ " eight minutes.", " twelve minutes.", " eighteen minutes.", " sixteen minutes." ], "tags": [ "rate" ], "prediction": 1, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -6.064818382263184, "mean_logprob": -2.021606127421061, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 3, "sum_logprob": -6.02801513671875, "mean_logprob": -2.00933837890625, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 4, "sum_logprob": -9.82884693145752, "mean_logprob": -2.45721173286438, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 4, "sum_logprob": -8.988325119018555, "mean_logprob": -2.2470812797546387, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.01226774851481105, "context_truncated": false, "elapsed_seconds": 0.031557569000142394 }, { "id": "quantitative-049", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 0.0, "item_elo": 1150, "context": "A number is increased by 8, doubled, and then reduced by 6 to give 30. The original number is", "continuations": [ " ten.", " eight.", " twelve.", " fourteen." ], "tags": [ "algebra" ], "prediction": 3, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -11.160680770874023, "mean_logprob": -5.580340385437012, "total_sequence_tokens": 28 }, { "index": 1, "tokens": 2, "sum_logprob": -8.79175853729248, "mean_logprob": -4.39587926864624, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 2, "sum_logprob": -11.474509239196777, "mean_logprob": -5.737254619598389, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 3, "sum_logprob": -10.697616577148438, "mean_logprob": -3.5658721923828125, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.8300070762634277, "context_truncated": false, "elapsed_seconds": 0.03413807600009022 }, { "id": "quantitative-050", "category": "quantitative", "difficulty": "hard", "category_weight": 1.3, "difficulty_weight": 2.25, "point_weight": 2.9250000000000003, "earned_points": 2.9250000000000003, "item_elo": 1150, "context": "A store marks an item up 25 percent from a cost of 64 euros. The marked price is", "continuations": [ " 72 euros.", " 80 euros.", " 84 euros.", " 89 euros." ], "tags": [ "percent" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -4.643354415893555, "mean_logprob": -0.6633363451276507, "total_sequence_tokens": 32 }, { "index": 1, "tokens": 7, "sum_logprob": -4.039778709411621, "mean_logprob": -0.5771112442016602, "total_sequence_tokens": 32 }, { "index": 2, "tokens": 7, "sum_logprob": -6.562363147735596, "mean_logprob": -0.9374804496765137, "total_sequence_tokens": 32 }, { "index": 3, "tokens": 7, "sum_logprob": -8.166106224060059, "mean_logprob": -1.1665866034371513, "total_sequence_tokens": 32 } ], "best_vs_second_margin": 0.08622510092599056, "context_truncated": false, "elapsed_seconds": 0.027850759999637376 }, { "id": "logical_reasoning-001", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 0.0, "item_elo": 950, "context": "All robins are birds. Pip is a robin. Therefore, Pip is", "continuations": [ " a bird.", " not a bird.", " a fish.", " necessarily blue." ], "tags": [ "deduction" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -5.120014190673828, "mean_logprob": -1.7066713968912761, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 4, "sum_logprob": -5.987151145935059, "mean_logprob": -1.4967877864837646, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 3, "sum_logprob": -5.111690044403076, "mean_logprob": -1.7038966814676921, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 3, "sum_logprob": -17.19256591796875, "mean_logprob": -5.730855305989583, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 0.20710889498392748, "context_truncated": false, "elapsed_seconds": 0.028051785000570817 }, { "id": "logical_reasoning-002", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 0.0, "item_elo": 950, "context": "No squares are circles. Shape K is a square. Shape K is therefore", "continuations": [ " a circle.", " not a square.", " both shapes.", " not a circle." ], "tags": [ "deduction" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -2.540952205657959, "mean_logprob": -0.846984068552653, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 4, "sum_logprob": -5.215645790100098, "mean_logprob": -1.3039114475250244, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 3, "sum_logprob": -14.818357467651367, "mean_logprob": -4.939452489217122, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 4, "sum_logprob": -6.576274871826172, "mean_logprob": -1.644068717956543, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.45692737897237146, "context_truncated": false, "elapsed_seconds": 0.026913691999652656 }, { "id": "logical_reasoning-003", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 0.0, "item_elo": 950, "context": "If the switch is on, the lamp glows. The switch is on, so the lamp", "continuations": [ " must be broken.", " is off.", " glows.", " has no switch." ], "tags": [ "modus_ponens" ], "prediction": 1, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -11.94361686706543, "mean_logprob": -2.9859042167663574, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 3, "sum_logprob": -2.789653778076172, "mean_logprob": -0.9298845926920573, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 3, "sum_logprob": -3.650395154953003, "mean_logprob": -1.2167983849843342, "total_sequence_tokens": 22 }, { "index": 3, "tokens": 4, "sum_logprob": -19.535369873046875, "mean_logprob": -4.883842468261719, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.286913792292277, "context_truncated": false, "elapsed_seconds": 0.027348768000592827 }, { "id": "logical_reasoning-004", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 0.0, "item_elo": 950, "context": "A is taller than B, and B is taller than C. It follows that", "continuations": [ " C is taller than A.", " A and C are equal.", " B is shortest.", " A is taller than C." ], "tags": [ "transitive" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -3.1261911392211914, "mean_logprob": -0.4465987341744559, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 6, "sum_logprob": -10.728402137756348, "mean_logprob": -1.7880670229593914, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 4, "sum_logprob": -14.119539260864258, "mean_logprob": -3.5298848152160645, "total_sequence_tokens": 22 }, { "index": 3, "tokens": 7, "sum_logprob": -5.211345195770264, "mean_logprob": -0.7444778851100377, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.29787915093558176, "context_truncated": false, "elapsed_seconds": 0.027715070000340347 }, { "id": "logical_reasoning-005", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 1.35, "item_elo": 950, "context": "Either the coin is in the cup or in the box. It is not in the cup. Thus it is", "continuations": [ " in the box.", " in both places.", " nowhere.", " still in the cup." ], "tags": [ "disjunction" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -4.2625555992126465, "mean_logprob": -1.0656388998031616, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 4, "sum_logprob": -11.732442855834961, "mean_logprob": -2.9331107139587402, "total_sequence_tokens": 27 }, { "index": 2, "tokens": 3, "sum_logprob": -21.61458969116211, "mean_logprob": -7.20486323038737, "total_sequence_tokens": 26 }, { "index": 3, "tokens": 5, "sum_logprob": -8.50075912475586, "mean_logprob": -1.7001518249511718, "total_sequence_tokens": 28 } ], "best_vs_second_margin": 0.6345129251480102, "context_truncated": false, "elapsed_seconds": 0.027197057999728713 }, { "id": "logical_reasoning-006", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 1.35, "item_elo": 950, "context": "Every metal key conducts electricity. This object is a metal key, so it", "continuations": [ " cannot be metal.", " conducts electricity.", " must be wooden.", " proves all objects conduct." ], "tags": [ "deduction" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -16.674589157104492, "mean_logprob": -4.168647289276123, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 4, "sum_logprob": -4.755519866943359, "mean_logprob": -1.1888799667358398, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 4, "sum_logprob": -14.105069160461426, "mean_logprob": -3.5262672901153564, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 5, "sum_logprob": -37.62774658203125, "mean_logprob": -7.52554931640625, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 2.3373873233795166, "context_truncated": false, "elapsed_seconds": 0.027393565998863778 }, { "id": "logical_reasoning-007", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 1.35, "item_elo": 950, "context": "If today is Monday, tomorrow is Tuesday. Today is Monday. Tomorrow is", "continuations": [ " Sunday.", " Wednesday.", " Tuesday.", " Monday." ], "tags": [ "conditional" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -4.646546363830566, "mean_logprob": -2.323273181915283, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 2, "sum_logprob": -2.154924154281616, "mean_logprob": -1.077462077140808, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 2, "sum_logprob": -0.5209131240844727, "mean_logprob": -0.26045656204223633, "total_sequence_tokens": 18 }, { "index": 3, "tokens": 2, "sum_logprob": -4.987026214599609, "mean_logprob": -2.4935131072998047, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 0.8170055150985718, "context_truncated": false, "elapsed_seconds": 0.03134048799984157 }, { "id": "logical_reasoning-008", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 0.0, "item_elo": 950, "context": "No red token is large. Token P is red. Token P is", "continuations": [ " definitely large.", " both red and large.", " not red.", " not large." ], "tags": [ "deduction" ], "prediction": 2, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -14.585792541503906, "mean_logprob": -4.861930847167969, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 5, "sum_logprob": -10.64543342590332, "mean_logprob": -2.1290866851806642, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 3, "sum_logprob": -2.6387014389038086, "mean_logprob": -0.8795671463012695, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 3, "sum_logprob": -6.280209541320801, "mean_logprob": -2.093403180440267, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 1.2138360341389975, "context_truncated": false, "elapsed_seconds": 0.027026356001442764 }, { "id": "logical_reasoning-009", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 1.35, "item_elo": 950, "context": "Only one of Door A and Door B is open. Door A is closed, so", "continuations": [ " Door B is open.", " both are closed.", " both are open.", " Door B must be closed." ], "tags": [ "exclusive_or" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -2.959444761276245, "mean_logprob": -0.49324079354604083, "total_sequence_tokens": 26 }, { "index": 1, "tokens": 4, "sum_logprob": -12.22856330871582, "mean_logprob": -3.057140827178955, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 4, "sum_logprob": -10.97695541381836, "mean_logprob": -2.74423885345459, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 7, "sum_logprob": -4.827787399291992, "mean_logprob": -0.6896839141845703, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.19644312063852948, "context_truncated": false, "elapsed_seconds": 0.02870630799952778 }, { "id": "logical_reasoning-010", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 1.35, "item_elo": 950, "context": "All cedar trees are plants. No plant is a machine. Therefore, no cedar tree is", "continuations": [ " a plant.", " a machine.", " a tree.", " alive." ], "tags": [ "syllogism" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -7.419729232788086, "mean_logprob": -2.473243077596029, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 3, "sum_logprob": -0.6436737775802612, "mean_logprob": -0.21455792586008707, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 3, "sum_logprob": -11.101336479187012, "mean_logprob": -3.7004454930623374, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 2, "sum_logprob": -7.84523868560791, "mean_logprob": -3.922619342803955, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 2.258685151735942, "context_truncated": false, "elapsed_seconds": 0.026393262000055984 }, { "id": "logical_reasoning-011", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 1.35, "item_elo": 950, "context": "If it rains, the path gets wet. The path is not wet. Therefore, it", "continuations": [ " certainly snowed.", " must be night.", " did not rain.", " is still raining." ], "tags": [ "modus_tollens" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -18.82676887512207, "mean_logprob": -4.706692218780518, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 4, "sum_logprob": -10.50233268737793, "mean_logprob": -2.6255831718444824, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 4, "sum_logprob": -6.35506010055542, "mean_logprob": -1.588765025138855, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 5, "sum_logprob": -9.369444847106934, "mean_logprob": -1.8738889694213867, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 0.28512394428253174, "context_truncated": false, "elapsed_seconds": 0.02827742499903252 }, { "id": "logical_reasoning-012", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 0.0, "item_elo": 950, "context": "L is before M, and M is before N. In this ordering, N is", "continuations": [ " first.", " before L.", " between L and M.", " after L." ], "tags": [ "order" ], "prediction": 2, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -5.804980278015137, "mean_logprob": -2.9024901390075684, "total_sequence_tokens": 19 }, { "index": 1, "tokens": 3, "sum_logprob": -9.521125793457031, "mean_logprob": -3.1737085978190103, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 5, "sum_logprob": -10.34437084197998, "mean_logprob": -2.068874168395996, "total_sequence_tokens": 22 }, { "index": 3, "tokens": 3, "sum_logprob": -9.500274658203125, "mean_logprob": -3.1667582194010415, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.8336159706115724, "context_truncated": false, "elapsed_seconds": 0.028135656999438652 }, { "id": "logical_reasoning-013", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 0.0, "item_elo": 950, "context": "Some poets are teachers. Every teacher reads. Therefore, some poets", "continuations": [ " read.", " never read.", " are not teachers.", " are all teachers." ], "tags": [ "syllogism" ], "prediction": 2, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -5.698014736175537, "mean_logprob": -2.8490073680877686, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 3, "sum_logprob": -7.821164131164551, "mean_logprob": -2.6070547103881836, "total_sequence_tokens": 18 }, { "index": 2, "tokens": 4, "sum_logprob": -6.846439361572266, "mean_logprob": -1.7116098403930664, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 4, "sum_logprob": -8.271710395812988, "mean_logprob": -2.067927598953247, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.35631775856018066, "context_truncated": false, "elapsed_seconds": 0.03454536499884853 }, { "id": "logical_reasoning-014", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 1.35, "item_elo": 950, "context": "Exactly one card is marked. Card X is not marked and Card Y is marked. Card Z is therefore", "continuations": [ " also marked.", " not marked.", " the same as Y.", " impossible to classify." ], "tags": [ "constraint" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -5.6638665199279785, "mean_logprob": -1.8879555066426594, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 3, "sum_logprob": -1.8135178089141846, "mean_logprob": -0.6045059363047282, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 5, "sum_logprob": -19.507362365722656, "mean_logprob": -3.901472473144531, "total_sequence_tokens": 26 }, { "index": 3, "tokens": 5, "sum_logprob": -23.16461181640625, "mean_logprob": -4.63292236328125, "total_sequence_tokens": 26 } ], "best_vs_second_margin": 1.2834495703379312, "context_truncated": false, "elapsed_seconds": 0.027377290000003995 }, { "id": "logical_reasoning-015", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 1.35, "item_elo": 950, "context": "If a number is divisible by four, it is even. Twenty is divisible by four, so twenty is", "continuations": [ " odd.", " prime.", " even.", " negative." ], "tags": [ "deduction" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -6.139820098876953, "mean_logprob": -3.0699100494384766, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 2, "sum_logprob": -10.674606323242188, "mean_logprob": -5.337303161621094, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 2, "sum_logprob": -1.3192028999328613, "mean_logprob": -0.6596014499664307, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 2, "sum_logprob": -15.370566368103027, "mean_logprob": -7.685283184051514, "total_sequence_tokens": 24 } ], "best_vs_second_margin": 2.410308599472046, "context_truncated": false, "elapsed_seconds": 0.025267508999604615 }, { "id": "logical_reasoning-016", "category": "logical_reasoning", "difficulty": "easy", "category_weight": 1.35, "difficulty_weight": 1.0, "point_weight": 1.35, "earned_points": 0.0, "item_elo": 950, "context": "No blue item is fragile. Vase V is fragile. Therefore, Vase V is", "continuations": [ " definitely blue.", " both blue and fragile.", " not fragile.", " not blue." ], "tags": [ "contraposition" ], "prediction": 2, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -13.683470726013184, "mean_logprob": -4.5611569086710615, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 6, "sum_logprob": -9.270719528198242, "mean_logprob": -1.5451199213663738, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 4, "sum_logprob": -4.348690032958984, "mean_logprob": -1.087172508239746, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 3, "sum_logprob": -7.262202262878418, "mean_logprob": -2.420734087626139, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.4579474131266277, "context_truncated": false, "elapsed_seconds": 0.02956127200013725 }, { "id": "logical_reasoning-017", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 0.0, "item_elo": 1050, "context": "All glims are nords. Some nords are pelts. From these facts alone, it follows that", "continuations": [ " no conclusion links glims to pelts.", " all glims are pelts.", " some glims are pelts.", " no nords are glims." ], "tags": [ "quantifiers" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 9, "sum_logprob": -37.376155853271484, "mean_logprob": -4.152906205919054, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 7, "sum_logprob": -13.048042297363281, "mean_logprob": -1.8640060424804688, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 7, "sum_logprob": -16.346662521362305, "mean_logprob": -2.335237503051758, "total_sequence_tokens": 29 }, { "index": 3, "tokens": 7, "sum_logprob": -17.042264938354492, "mean_logprob": -2.4346092769077847, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.47123146057128906, "context_truncated": false, "elapsed_seconds": 0.0377511910010071 }, { "id": "logical_reasoning-018", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "If R then S. If S then T. R is true. Therefore,", "continuations": [ " T is false.", " T is true.", " S is false.", " R is false." ], "tags": [ "conditional_chain" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -3.9769163131713867, "mean_logprob": -0.9942290782928467, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 4, "sum_logprob": -3.0385985374450684, "mean_logprob": -0.7596496343612671, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 4, "sum_logprob": -5.500736236572266, "mean_logprob": -1.3751840591430664, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 4, "sum_logprob": -6.0516676902771, "mean_logprob": -1.512916922569275, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.2345794439315796, "context_truncated": false, "elapsed_seconds": 0.035311272000399185 }, { "id": "logical_reasoning-019", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 0.0, "item_elo": 1050, "context": "Exactly two of A, B, and C are true. A is true and B is false. Therefore, C is", "continuations": [ " false.", " unknown.", " true.", " both true and false." ], "tags": [ "constraint" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -0.36741650104522705, "mean_logprob": -0.18370825052261353, "total_sequence_tokens": 26 }, { "index": 1, "tokens": 2, "sum_logprob": -10.698878288269043, "mean_logprob": -5.3494391441345215, "total_sequence_tokens": 26 }, { "index": 2, "tokens": 2, "sum_logprob": -1.841135859489441, "mean_logprob": -0.9205679297447205, "total_sequence_tokens": 26 }, { "index": 3, "tokens": 5, "sum_logprob": -7.60679292678833, "mean_logprob": -1.521358585357666, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.7368596792221069, "context_truncated": false, "elapsed_seconds": 0.03049728999940271 }, { "id": "logical_reasoning-020", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 0.0, "item_elo": 1050, "context": "Kai sits somewhere left of Lio. Mina sits right of Lio. Which statement must be true?", "continuations": [ " Mina sits left of Kai.", " Kai sits beside Mina.", " Lio sits left of Kai.", " Kai sits left of Mina." ], "tags": [ "spatial" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 9, "sum_logprob": -20.981042861938477, "mean_logprob": -2.3312269846598306, "total_sequence_tokens": 34 }, { "index": 1, "tokens": 9, "sum_logprob": -22.95328140258789, "mean_logprob": -2.5503646002875433, "total_sequence_tokens": 34 }, { "index": 2, "tokens": 9, "sum_logprob": -21.372692108154297, "mean_logprob": -2.3747435675726996, "total_sequence_tokens": 34 }, { "index": 3, "tokens": 9, "sum_logprob": -21.874128341674805, "mean_logprob": -2.4304587046305337, "total_sequence_tokens": 34 } ], "best_vs_second_margin": 0.04351658291286897, "context_truncated": false, "elapsed_seconds": 0.03377929599992058 }, { "id": "logical_reasoning-021", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "Every zor is a vek, and no vek is a lum. Therefore,", "continuations": [ " no zor is a lum.", " every lum is a zor.", " some zors are lums.", " no zor is a vek." ], "tags": [ "syllogism" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -9.927262306213379, "mean_logprob": -1.2409077882766724, "total_sequence_tokens": 27 }, { "index": 1, "tokens": 8, "sum_logprob": -12.893324851989746, "mean_logprob": -1.6116656064987183, "total_sequence_tokens": 27 }, { "index": 2, "tokens": 7, "sum_logprob": -23.006277084350586, "mean_logprob": -3.2866110120500838, "total_sequence_tokens": 26 }, { "index": 3, "tokens": 8, "sum_logprob": -12.96645736694336, "mean_logprob": -1.62080717086792, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.3707578182220459, "context_truncated": false, "elapsed_seconds": 0.029061327000817982 }, { "id": "logical_reasoning-022", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 0.0, "item_elo": 1050, "context": "The alarm rings if and only if the sensor is active. The alarm is silent. Thus the sensor is", "continuations": [ " active.", " inactive.", " both states.", " unrelated to the alarm." ], "tags": [ "biconditional" ], "prediction": 3, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -3.9675488471984863, "mean_logprob": -1.9837744235992432, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 2, "sum_logprob": -4.27786111831665, "mean_logprob": -2.138930559158325, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 3, "sum_logprob": -24.123218536376953, "mean_logprob": -8.041072845458984, "total_sequence_tokens": 24 }, { "index": 3, "tokens": 6, "sum_logprob": -11.43774127960205, "mean_logprob": -1.9062902132670085, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.07748421033223463, "context_truncated": false, "elapsed_seconds": 0.026720802999989246 }, { "id": "logical_reasoning-023", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "At least one of P or Q is false. P is true. It follows that Q is", "continuations": [ " true.", " undetermined.", " false.", " equal to P." ], "tags": [ "propositional" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -1.5732948780059814, "mean_logprob": -0.7866474390029907, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 4, "sum_logprob": -17.106487274169922, "mean_logprob": -4.2766218185424805, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 2, "sum_logprob": -0.8197979927062988, "mean_logprob": -0.4098989963531494, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 4, "sum_logprob": -10.352224349975586, "mean_logprob": -2.5880560874938965, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 0.3767484426498413, "context_truncated": false, "elapsed_seconds": 0.025991026999690803 }, { "id": "logical_reasoning-024", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "Three runners finish with no ties. Ada is not first. Bo finishes after Cy. The only possible first-place runner is", "continuations": [ " Ada.", " Bo.", " either Ada or Bo.", " Cy." ], "tags": [ "constraint" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -7.815156936645508, "mean_logprob": -3.907578468322754, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 2, "sum_logprob": -7.207143783569336, "mean_logprob": -3.603571891784668, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 5, "sum_logprob": -18.599592208862305, "mean_logprob": -3.719918441772461, "total_sequence_tokens": 32 }, { "index": 3, "tokens": 2, "sum_logprob": -6.466699123382568, "mean_logprob": -3.233349561691284, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.3702223300933838, "context_truncated": false, "elapsed_seconds": 0.026135096999496454 }, { "id": "logical_reasoning-025", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "If the report is approved, funding continues. Funding did not continue. Therefore, the report was", "continuations": [ " not approved.", " approved late.", " necessarily reviewed.", " unrelated to funding." ], "tags": [ "modus_tollens" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -5.09077262878418, "mean_logprob": -1.6969242095947266, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 3, "sum_logprob": -17.806264877319336, "mean_logprob": -5.935421625773112, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 3, "sum_logprob": -21.076892852783203, "mean_logprob": -7.025630950927734, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 5, "sum_logprob": -18.95391273498535, "mean_logprob": -3.7907825469970704, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 2.093858337402344, "context_truncated": false, "elapsed_seconds": 0.06509407600060513 }, { "id": "logical_reasoning-026", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "Some artists are engineers, and no engineer is careless. Therefore, some artists are", "continuations": [ " careless.", " not careless.", " not engineers.", " all engineers." ], "tags": [ "syllogism" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -6.2993855476379395, "mean_logprob": -2.09979518254598, "total_sequence_tokens": 20 }, { "index": 1, "tokens": 4, "sum_logprob": -7.499154090881348, "mean_logprob": -1.874788522720337, "total_sequence_tokens": 21 }, { "index": 2, "tokens": 3, "sum_logprob": -5.7073564529418945, "mean_logprob": -1.9024521509806316, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 3, "sum_logprob": -9.565094947814941, "mean_logprob": -3.1883649826049805, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.02766362826029467, "context_truncated": false, "elapsed_seconds": 0.06115498600047431 }, { "id": "logical_reasoning-027", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "A statement and its negation cannot both be true. The statement is true, so its negation is", "continuations": [ " true.", " unknown.", " false.", " equivalent." ], "tags": [ "negation" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -4.395155906677246, "mean_logprob": -2.197577953338623, "total_sequence_tokens": 21 }, { "index": 1, "tokens": 2, "sum_logprob": -14.564260482788086, "mean_logprob": -7.282130241394043, "total_sequence_tokens": 21 }, { "index": 2, "tokens": 2, "sum_logprob": -0.576337456703186, "mean_logprob": -0.288168728351593, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 2, "sum_logprob": -14.92807674407959, "mean_logprob": -7.464038372039795, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 1.90940922498703, "context_truncated": false, "elapsed_seconds": 0.0335494060000201 }, { "id": "logical_reasoning-028", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 0.0, "item_elo": 1050, "context": "If X is selected, Y must be selected. If Y is selected, Z cannot be selected. X is selected, so Z", "continuations": [ " must be selected.", " may equal X.", " makes Y unselected.", " cannot be selected." ], "tags": [ "constraint" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -1.6457048654556274, "mean_logprob": -0.41142621636390686, "total_sequence_tokens": 30 }, { "index": 1, "tokens": 4, "sum_logprob": -17.842538833618164, "mean_logprob": -4.460634708404541, "total_sequence_tokens": 30 }, { "index": 2, "tokens": 5, "sum_logprob": -15.62555980682373, "mean_logprob": -3.125111961364746, "total_sequence_tokens": 31 }, { "index": 3, "tokens": 4, "sum_logprob": -4.505889415740967, "mean_logprob": -1.1264723539352417, "total_sequence_tokens": 30 } ], "best_vs_second_margin": 0.7150461375713348, "context_truncated": false, "elapsed_seconds": 0.03742306400090456 }, { "id": "logical_reasoning-029", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 0.0, "item_elo": 1050, "context": "No historians are timeless beings. Some writers are historians. Therefore,", "continuations": [ " some writers are not timeless beings.", " all writers are timeless.", " no writers are historians.", " all historians are writers." ], "tags": [ "syllogism" ], "prediction": 3, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -24.91695785522461, "mean_logprob": -3.114619731903076, "total_sequence_tokens": 22 }, { "index": 1, "tokens": 6, "sum_logprob": -15.53573989868164, "mean_logprob": -2.589289983113607, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 5, "sum_logprob": -12.175522804260254, "mean_logprob": -2.435104560852051, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 5, "sum_logprob": -11.63157844543457, "mean_logprob": -2.326315689086914, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.10878887176513707, "context_truncated": false, "elapsed_seconds": 0.028489014999649953 }, { "id": "logical_reasoning-030", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "Either J or K, but not both, is open. If J is open, L is closed. J is open. Hence", "continuations": [ " K and L are open.", " K and L are closed.", " K is open and L closed.", " K is closed and L open." ], "tags": [ "propositional" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -11.102985382080078, "mean_logprob": -1.850497563680013, "total_sequence_tokens": 33 }, { "index": 1, "tokens": 6, "sum_logprob": -10.669707298278809, "mean_logprob": -1.7782845497131348, "total_sequence_tokens": 33 }, { "index": 2, "tokens": 7, "sum_logprob": -13.543540954589844, "mean_logprob": -1.9347915649414062, "total_sequence_tokens": 34 }, { "index": 3, "tokens": 7, "sum_logprob": -15.956045150756836, "mean_logprob": -2.279435021536691, "total_sequence_tokens": 34 } ], "best_vs_second_margin": 0.07221301396687818, "context_truncated": false, "elapsed_seconds": 0.039278928999920026 }, { "id": "logical_reasoning-031", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "Rhea is before Sami. Taro is after Sami. Ula is before Rhea. The second person in order is", "continuations": [ " Ula.", " Sami.", " Rhea.", " Taro." ], "tags": [ "order" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -3.3497650623321533, "mean_logprob": -1.1165883541107178, "total_sequence_tokens": 30 }, { "index": 1, "tokens": 3, "sum_logprob": -2.953437566757202, "mean_logprob": -0.9844791889190674, "total_sequence_tokens": 30 }, { "index": 2, "tokens": 3, "sum_logprob": -2.262697458267212, "mean_logprob": -0.7542324860890707, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 3, "sum_logprob": -2.8338801860809326, "mean_logprob": -0.9446267286936442, "total_sequence_tokens": 30 } ], "best_vs_second_margin": 0.1903942426045735, "context_truncated": false, "elapsed_seconds": 0.02460370500011777 }, { "id": "logical_reasoning-032", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "All valid tickets have a stamp. This ticket has no stamp. It is therefore", "continuations": [ " valid.", " stamped invisibly.", " both valid and invalid.", " not valid." ], "tags": [ "contraposition" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -2.1518657207489014, "mean_logprob": -1.0759328603744507, "total_sequence_tokens": 18 }, { "index": 1, "tokens": 6, "sum_logprob": -25.669761657714844, "mean_logprob": -4.278293609619141, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 5, "sum_logprob": -8.00202751159668, "mean_logprob": -1.600405502319336, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 3, "sum_logprob": -2.469339370727539, "mean_logprob": -0.8231131235758463, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.25281973679860437, "context_truncated": false, "elapsed_seconds": 0.02711293900028977 }, { "id": "logical_reasoning-033", "category": "logical_reasoning", "difficulty": "medium", "category_weight": 1.35, "difficulty_weight": 1.5, "point_weight": 2.0250000000000004, "earned_points": 2.0250000000000004, "item_elo": 1050, "context": "Exactly one of the claims \"A is true\" and \"B is false\" is true. A is false. Therefore, B is", "continuations": [ " false.", " true.", " both.", " undetermined." ], "tags": [ "propositional" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -0.25394731760025024, "mean_logprob": -0.12697365880012512, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 2, "sum_logprob": -3.023630380630493, "mean_logprob": -1.5118151903152466, "total_sequence_tokens": 31 }, { "index": 2, "tokens": 2, "sum_logprob": -17.4687557220459, "mean_logprob": -8.73437786102295, "total_sequence_tokens": 31 }, { "index": 3, "tokens": 4, "sum_logprob": -17.809072494506836, "mean_logprob": -4.452268123626709, "total_sequence_tokens": 33 } ], "best_vs_second_margin": 1.3848415315151215, "context_truncated": false, "elapsed_seconds": 0.03756342099950416 }, { "id": "logical_reasoning-034", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "Every quill is a renn. No renn is a soth. Some varks are quills. It follows that some varks are", "continuations": [ " soths.", " not soths.", " not quills.", " all renns." ], "tags": [ "syllogism" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 3, "sum_logprob": -5.736642837524414, "mean_logprob": -1.9122142791748047, "total_sequence_tokens": 33 }, { "index": 1, "tokens": 4, "sum_logprob": -8.529621124267578, "mean_logprob": -2.1324052810668945, "total_sequence_tokens": 34 }, { "index": 2, "tokens": 4, "sum_logprob": -3.5806849002838135, "mean_logprob": -0.8951712250709534, "total_sequence_tokens": 34 }, { "index": 3, "tokens": 4, "sum_logprob": -20.263328552246094, "mean_logprob": -5.065832138061523, "total_sequence_tokens": 34 } ], "best_vs_second_margin": 1.0170430541038513, "context_truncated": false, "elapsed_seconds": 0.037289870999302366 }, { "id": "logical_reasoning-035", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "Exactly two of P, Q, R, and S are true. P and Q have opposite truth values. R is true and S is false. Therefore,", "continuations": [ " P and Q are both true.", " P and Q are both false.", " exactly one of P and Q is true.", " neither P nor Q matters." ], "tags": [ "constraint" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -6.264081954956055, "mean_logprob": -0.8948688507080078, "total_sequence_tokens": 39 }, { "index": 1, "tokens": 7, "sum_logprob": -6.364994525909424, "mean_logprob": -0.9092849322727748, "total_sequence_tokens": 39 }, { "index": 2, "tokens": 9, "sum_logprob": -19.117538452148438, "mean_logprob": -2.124170939127604, "total_sequence_tokens": 41 }, { "index": 3, "tokens": 6, "sum_logprob": -13.39778995513916, "mean_logprob": -2.2329649925231934, "total_sequence_tokens": 38 } ], "best_vs_second_margin": 0.014416081564767036, "context_truncated": false, "elapsed_seconds": 0.03755214200100454 }, { "id": "logical_reasoning-036", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "If A then B. If not C then not B. A is true. Which conclusion must follow?", "continuations": [ " C is false.", " B is false.", " A is false.", " C is true." ], "tags": [ "conditional_chain" ], "prediction": 2, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -17.453317642211914, "mean_logprob": -4.3633294105529785, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 4, "sum_logprob": -17.41801643371582, "mean_logprob": -4.354504108428955, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 4, "sum_logprob": -16.327856063842773, "mean_logprob": -4.081964015960693, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 4, "sum_logprob": -17.262537002563477, "mean_logprob": -4.315634250640869, "total_sequence_tokens": 25 } ], "best_vs_second_margin": 0.23367023468017578, "context_truncated": false, "elapsed_seconds": 0.027254909999101073 }, { "id": "logical_reasoning-037", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 3.0375, "item_elo": 1200, "context": "Five seats are numbered 1 through 5. Iva sits immediately left of Jae. Kian sits in seat 5. Iva sits in seat 2. Jae sits in", "continuations": [ " seat 3.", " seat 1.", " seat 4.", " seat 5." ], "tags": [ "constraint" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -1.0388476848602295, "mean_logprob": -0.2597119212150574, "total_sequence_tokens": 48 }, { "index": 1, "tokens": 4, "sum_logprob": -1.3396497964859009, "mean_logprob": -0.3349124491214752, "total_sequence_tokens": 48 }, { "index": 2, "tokens": 4, "sum_logprob": -1.67790687084198, "mean_logprob": -0.419476717710495, "total_sequence_tokens": 48 }, { "index": 3, "tokens": 4, "sum_logprob": -5.073331356048584, "mean_logprob": -1.268332839012146, "total_sequence_tokens": 48 } ], "best_vs_second_margin": 0.07520052790641785, "context_truncated": false, "elapsed_seconds": 0.03574124800070422 }, { "id": "logical_reasoning-038", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "No fal is a mer. Every tov is a fal. Some riks are mers. Which statement must be true?", "continuations": [ " Some riks are tovs.", " No tov is a mer.", " Every rik is a fal.", " Some mers are fals." ], "tags": [ "quantifiers" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -20.39111328125, "mean_logprob": -2.9130161830357144, "total_sequence_tokens": 35 }, { "index": 1, "tokens": 7, "sum_logprob": -18.03400230407715, "mean_logprob": -2.5762860434395924, "total_sequence_tokens": 35 }, { "index": 2, "tokens": 8, "sum_logprob": -15.599857330322266, "mean_logprob": -1.9499821662902832, "total_sequence_tokens": 36 }, { "index": 3, "tokens": 7, "sum_logprob": -17.913246154785156, "mean_logprob": -2.559035164969308, "total_sequence_tokens": 35 } ], "best_vs_second_margin": 0.609052998679025, "context_truncated": false, "elapsed_seconds": 0.0381310630000371 }, { "id": "logical_reasoning-039", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "A safe opens exactly when both keys are turned. The safe did not open, and the first key was turned. Therefore, the second key was", "continuations": [ " turned.", " broken necessarily.", " not turned.", " unrelated." ], "tags": [ "conjunction" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -3.114652633666992, "mean_logprob": -1.557326316833496, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 3, "sum_logprob": -24.938669204711914, "mean_logprob": -8.31288973490397, "total_sequence_tokens": 32 }, { "index": 2, "tokens": 3, "sum_logprob": -6.651601314544678, "mean_logprob": -2.217200438181559, "total_sequence_tokens": 32 }, { "index": 3, "tokens": 3, "sum_logprob": -13.41206169128418, "mean_logprob": -4.47068723042806, "total_sequence_tokens": 32 } ], "best_vs_second_margin": 0.659874121348063, "context_truncated": false, "elapsed_seconds": 0.025557093000315945 }, { "id": "logical_reasoning-040", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 3.0375, "item_elo": 1200, "context": "A committee selects exactly three of J, K, L, and M. If J is selected, K is not. L and M are selected. Therefore, the third member must be", "continuations": [ " J.", " L again.", " M again.", " K." ], "tags": [ "constraint" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -6.013801574707031, "mean_logprob": -3.0069007873535156, "total_sequence_tokens": 40 }, { "index": 1, "tokens": 3, "sum_logprob": -14.643722534179688, "mean_logprob": -4.8812408447265625, "total_sequence_tokens": 41 }, { "index": 2, "tokens": 3, "sum_logprob": -15.19642448425293, "mean_logprob": -5.06547482808431, "total_sequence_tokens": 41 }, { "index": 3, "tokens": 2, "sum_logprob": -4.991424560546875, "mean_logprob": -2.4957122802734375, "total_sequence_tokens": 40 } ], "best_vs_second_margin": 0.5111885070800781, "context_truncated": false, "elapsed_seconds": 0.03555398400021659 }, { "id": "logical_reasoning-041", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "Some nels are paks. All paks are wims. No wim is a dor. Therefore,", "continuations": [ " some nels are not dors.", " all nels are wims.", " some dors are paks.", " no nels are paks." ], "tags": [ "syllogism" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -17.275667190551758, "mean_logprob": -2.1594583988189697, "total_sequence_tokens": 32 }, { "index": 1, "tokens": 7, "sum_logprob": -7.622884750366211, "mean_logprob": -1.0889835357666016, "total_sequence_tokens": 31 }, { "index": 2, "tokens": 7, "sum_logprob": -15.269927978515625, "mean_logprob": -2.1814182826450894, "total_sequence_tokens": 31 }, { "index": 3, "tokens": 7, "sum_logprob": -11.179550170898438, "mean_logprob": -1.597078595842634, "total_sequence_tokens": 31 } ], "best_vs_second_margin": 0.5080950600760323, "context_truncated": false, "elapsed_seconds": 0.03802208500019333 }, { "id": "logical_reasoning-042", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "Four tasks W, X, Y, Z are scheduled once each. W precedes X. Y immediately precedes Z. X is first. These conditions are", "continuations": [ " satisfied by X, W, Y, Z.", " impossible to satisfy.", " satisfied by W, X, Y, Z.", " satisfied by Y, Z, W, X." ], "tags": [ "consistency" ], "prediction": 2, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 10, "sum_logprob": -18.201345443725586, "mean_logprob": -1.8201345443725585, "total_sequence_tokens": 42 }, { "index": 1, "tokens": 4, "sum_logprob": -10.938289642333984, "mean_logprob": -2.734572410583496, "total_sequence_tokens": 36 }, { "index": 2, "tokens": 10, "sum_logprob": -15.769869804382324, "mean_logprob": -1.5769869804382324, "total_sequence_tokens": 42 }, { "index": 3, "tokens": 10, "sum_logprob": -23.666305541992188, "mean_logprob": -2.3666305541992188, "total_sequence_tokens": 42 } ], "best_vs_second_margin": 0.24314756393432613, "context_truncated": false, "elapsed_seconds": 0.04649977599910926 }, { "id": "logical_reasoning-043", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 3.0375, "item_elo": 1200, "context": "If either M or N is true, P is true. P is false. Therefore,", "continuations": [ " M is true.", " N is true.", " both M and N are false.", " exactly one is false." ], "tags": [ "contraposition" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -9.543510437011719, "mean_logprob": -2.3858776092529297, "total_sequence_tokens": 22 }, { "index": 1, "tokens": 4, "sum_logprob": -9.155370712280273, "mean_logprob": -2.2888426780700684, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 7, "sum_logprob": -6.482165336608887, "mean_logprob": -0.9260236195155552, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 5, "sum_logprob": -15.393881797790527, "mean_logprob": -3.0787763595581055, "total_sequence_tokens": 23 } ], "best_vs_second_margin": 1.3628190585545132, "context_truncated": false, "elapsed_seconds": 0.026972736000971054 }, { "id": "logical_reasoning-044", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "All A-things are B-things. Some B-things are C-things. No C-thing is a D-thing. Which claim is guaranteed?", "continuations": [ " Some A-things are C-things.", " No A-thing is a D-thing.", " All B-things are A-things.", " Some B-things are not D-things." ], "tags": [ "quantifiers" ], "prediction": 2, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 11, "sum_logprob": -12.665159225463867, "mean_logprob": -1.1513781114058061, "total_sequence_tokens": 48 }, { "index": 1, "tokens": 10, "sum_logprob": -14.303797721862793, "mean_logprob": -1.4303797721862792, "total_sequence_tokens": 47 }, { "index": 2, "tokens": 11, "sum_logprob": -10.99329662322998, "mean_logprob": -0.9993906021118164, "total_sequence_tokens": 48 }, { "index": 3, "tokens": 12, "sum_logprob": -14.286218643188477, "mean_logprob": -1.1905182202657063, "total_sequence_tokens": 49 } ], "best_vs_second_margin": 0.15198750929398974, "context_truncated": false, "elapsed_seconds": 0.04841408700121974 }, { "id": "logical_reasoning-045", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 3.0375, "item_elo": 1200, "context": "A code is valid if exactly one of conditions X and Y holds. X holds and the code is valid. Therefore, Y", "continuations": [ " does not hold.", " also holds.", " is equivalent to X.", " cannot be discussed." ], "tags": [ "exclusive_or" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -3.2396998405456543, "mean_logprob": -0.8099249601364136, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 3, "sum_logprob": -4.657301425933838, "mean_logprob": -1.5524338086446126, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 5, "sum_logprob": -10.75133991241455, "mean_logprob": -2.15026798248291, "total_sequence_tokens": 30 }, { "index": 3, "tokens": 4, "sum_logprob": -20.620361328125, "mean_logprob": -5.15509033203125, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.742508848508199, "context_truncated": false, "elapsed_seconds": 0.031652417999794125 }, { "id": "logical_reasoning-046", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 3.0375, "item_elo": 1200, "context": "R is earlier than S. T is later than U. S is earlier than U. Which ordering must hold?", "continuations": [ " T is earlier than R.", " R is earlier than T.", " U is earlier than S.", " S is later than T." ], "tags": [ "transitive" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -20.732337951660156, "mean_logprob": -3.455389658610026, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 6, "sum_logprob": -14.730389595031738, "mean_logprob": -2.4550649325052896, "total_sequence_tokens": 29 }, { "index": 2, "tokens": 6, "sum_logprob": -20.681095123291016, "mean_logprob": -3.4468491872151694, "total_sequence_tokens": 29 }, { "index": 3, "tokens": 6, "sum_logprob": -16.98881721496582, "mean_logprob": -2.8314695358276367, "total_sequence_tokens": 29 } ], "best_vs_second_margin": 0.37640460332234715, "context_truncated": false, "elapsed_seconds": 0.05448604800039902 }, { "id": "logical_reasoning-047", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "If the engine starts, either fuel or battery power is available. The engine starts, and no fuel is available. Thus", "continuations": [ " no power is available.", " the engine did not start.", " battery power is available.", " fuel is available." ], "tags": [ "disjunction" ], "prediction": 0, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 5, "sum_logprob": -21.319812774658203, "mean_logprob": -4.2639625549316404, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 6, "sum_logprob": -25.933473587036133, "mean_logprob": -4.3222455978393555, "total_sequence_tokens": 30 }, { "index": 2, "tokens": 5, "sum_logprob": -23.158662796020508, "mean_logprob": -4.631732559204101, "total_sequence_tokens": 29 }, { "index": 3, "tokens": 4, "sum_logprob": -22.04351806640625, "mean_logprob": -5.5108795166015625, "total_sequence_tokens": 28 } ], "best_vs_second_margin": 0.05828304290771502, "context_truncated": false, "elapsed_seconds": 0.03828422100013995 }, { "id": "logical_reasoning-048", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "Exactly one person among A, B, and C lies. A says B tells the truth. B says C tells the truth. C says A lies. The liar is", "continuations": [ " A.", " B.", " impossible to determine.", " C." ], "tags": [ "truth_tellers" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -4.278550624847412, "mean_logprob": -2.139275312423706, "total_sequence_tokens": 37 }, { "index": 1, "tokens": 2, "sum_logprob": -8.535700798034668, "mean_logprob": -4.267850399017334, "total_sequence_tokens": 37 }, { "index": 2, "tokens": 4, "sum_logprob": -15.505983352661133, "mean_logprob": -3.876495838165283, "total_sequence_tokens": 39 }, { "index": 3, "tokens": 2, "sum_logprob": -11.253164291381836, "mean_logprob": -5.626582145690918, "total_sequence_tokens": 37 } ], "best_vs_second_margin": 1.7372205257415771, "context_truncated": false, "elapsed_seconds": 0.03660139399835316 }, { "id": "logical_reasoning-049", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 0.0, "item_elo": 1200, "context": "No glens are haps. Some jors are glens. Every jor is a mip. It follows that some mips are", "continuations": [ " not haps.", " haps.", " not jors.", " all glens." ], "tags": [ "syllogism" ], "prediction": 1, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -8.938095092773438, "mean_logprob": -2.2345237731933594, "total_sequence_tokens": 33 }, { "index": 1, "tokens": 3, "sum_logprob": -2.8486106395721436, "mean_logprob": -0.9495368798573812, "total_sequence_tokens": 32 }, { "index": 2, "tokens": 4, "sum_logprob": -8.137388229370117, "mean_logprob": -2.0343470573425293, "total_sequence_tokens": 33 }, { "index": 3, "tokens": 4, "sum_logprob": -5.661764144897461, "mean_logprob": -1.4154410362243652, "total_sequence_tokens": 33 } ], "best_vs_second_margin": 0.465904156366984, "context_truncated": false, "elapsed_seconds": 0.03652822299955005 }, { "id": "logical_reasoning-050", "category": "logical_reasoning", "difficulty": "hard", "category_weight": 1.35, "difficulty_weight": 2.25, "point_weight": 3.0375, "earned_points": 3.0375, "item_elo": 1200, "context": "A proposition is true only if Q is true, and Q is true only if R is false. The proposition is true. Therefore, R is", "continuations": [ " true.", " false.", " both true and false.", " independent of Q." ], "tags": [ "conditional_chain" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 2, "sum_logprob": -2.581470012664795, "mean_logprob": -1.2907350063323975, "total_sequence_tokens": 31 }, { "index": 1, "tokens": 2, "sum_logprob": -0.4033554494380951, "mean_logprob": -0.20167772471904755, "total_sequence_tokens": 31 }, { "index": 2, "tokens": 5, "sum_logprob": -5.504075527191162, "mean_logprob": -1.1008151054382325, "total_sequence_tokens": 34 }, { "index": 3, "tokens": 4, "sum_logprob": -9.2425537109375, "mean_logprob": -2.310638427734375, "total_sequence_tokens": 33 } ], "best_vs_second_margin": 0.8991373807191849, "context_truncated": false, "elapsed_seconds": 0.03524178599946026 }, { "id": "code_completion-001", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def square(x):\n", "continuations": [ " return x * x\n", " return x + x\n", " print(x)\n", " return 2\n" ], "tags": [ "python" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -5.934330463409424, "mean_logprob": -0.8477614947727748, "total_sequence_tokens": 13 }, { "index": 1, "tokens": 7, "sum_logprob": -14.469461441040039, "mean_logprob": -2.067065920148577, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 6, "sum_logprob": -9.781411170959473, "mean_logprob": -1.630235195159912, "total_sequence_tokens": 12 }, { "index": 3, "tokens": 5, "sum_logprob": -18.043697357177734, "mean_logprob": -3.608739471435547, "total_sequence_tokens": 11 } ], "best_vs_second_margin": 0.7824737003871373, "context_truncated": false, "elapsed_seconds": 0.026375286000984488 }, { "id": "code_completion-002", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def is_empty(items):\n", "continuations": [ " return items[0]\n", " return len(items) == 0\n", " return len(items) + 1\n", " items.clear()\n" ], "tags": [ "python" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -14.702980995178223, "mean_logprob": -2.1004258564540317, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 11, "sum_logprob": -8.024712562561035, "mean_logprob": -0.729519323869185, "total_sequence_tokens": 19 }, { "index": 2, "tokens": 11, "sum_logprob": -16.210472106933594, "mean_logprob": -1.4736792824485085, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 6, "sum_logprob": -9.871957778930664, "mean_logprob": -1.645326296488444, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 0.7441599585793235, "context_truncated": false, "elapsed_seconds": 0.0282552590015257 }, { "id": "code_completion-003", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def greet(name):\n", "continuations": [ " return name - \"Hello\"\n", " print = name\n", " return \"Hello, \" + name\n", " return len(\"Hello\")\n" ], "tags": [ "python" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 10, "sum_logprob": -24.306949615478516, "mean_logprob": -2.4306949615478515, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 6, "sum_logprob": -19.574186325073242, "mean_logprob": -3.262364387512207, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 12, "sum_logprob": -11.44089412689209, "mean_logprob": -0.9534078439076742, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 7, "sum_logprob": -27.81157684326172, "mean_logprob": -3.9730824061802457, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 1.4772871176401772, "context_truncated": false, "elapsed_seconds": 0.03970513099920936 }, { "id": "code_completion-004", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def first(items):\n", "continuations": [ " return items[-1]\n", " return len(items)\n", " return items\n", " return items[0]\n" ], "tags": [ "python" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -15.152753829956055, "mean_logprob": -2.1646791185651506, "total_sequence_tokens": 13 }, { "index": 1, "tokens": 7, "sum_logprob": -14.308820724487305, "mean_logprob": -2.0441172463553294, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 4, "sum_logprob": -9.552918434143066, "mean_logprob": -2.3882296085357666, "total_sequence_tokens": 10 }, { "index": 3, "tokens": 7, "sum_logprob": -11.064523696899414, "mean_logprob": -1.580646242414202, "total_sequence_tokens": 13 } ], "best_vs_second_margin": 0.4634710039411274, "context_truncated": false, "elapsed_seconds": 0.027532070000233944 }, { "id": "code_completion-005", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def add(a, b):\n", "continuations": [ " return a + b\n", " return a - b\n", " return a == b\n", " return [a, b]\n" ], "tags": [ "python" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -8.011276245117188, "mean_logprob": -1.144468035016741, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 7, "sum_logprob": -17.55638313293457, "mean_logprob": -2.508054733276367, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 7, "sum_logprob": -19.319067001342773, "mean_logprob": -2.759866714477539, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 9, "sum_logprob": -14.94140625, "mean_logprob": -1.66015625, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 0.5156882149832589, "context_truncated": false, "elapsed_seconds": 0.026762985999084776 }, { "id": "code_completion-006", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def last(items):\n", "continuations": [ " return items[0]\n", " return items[-1]\n", " return items[1:]\n", " return -len(items)\n" ], "tags": [ "python" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -12.57072639465332, "mean_logprob": -1.7958180563790458, "total_sequence_tokens": 13 }, { "index": 1, "tokens": 7, "sum_logprob": -10.278488159179688, "mean_logprob": -1.468355451311384, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 7, "sum_logprob": -13.712234497070312, "mean_logprob": -1.958890642438616, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 9, "sum_logprob": -18.184860229492188, "mean_logprob": -2.020540025499132, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 0.3274626050676619, "context_truncated": false, "elapsed_seconds": 0.03602598600082274 }, { "id": "code_completion-007", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def is_even(n):\n", "continuations": [ " return n / 2\n", " return n % 2 == 1\n", " return n % 2 == 0\n", " return n * 2\n" ], "tags": [ "python" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -24.60840606689453, "mean_logprob": -3.0760507583618164, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 12, "sum_logprob": -19.844167709350586, "mean_logprob": -1.653680642445882, "total_sequence_tokens": 20 }, { "index": 2, "tokens": 12, "sum_logprob": -6.369754791259766, "mean_logprob": -0.5308128992716471, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 8, "sum_logprob": -24.57311248779297, "mean_logprob": -3.071639060974121, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 1.1228677431742349, "context_truncated": false, "elapsed_seconds": 0.02552455099976214 }, { "id": "code_completion-008", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def absolute_value(x):\n", "continuations": [ " return -abs(x)\n", " return x * 0\n", " return bool(x)\n", " return abs(x)\n" ], "tags": [ "python" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 9, "sum_logprob": -11.221795082092285, "mean_logprob": -1.2468661202324762, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 8, "sum_logprob": -19.985536575317383, "mean_logprob": -2.498192071914673, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 7, "sum_logprob": -15.934894561767578, "mean_logprob": -2.2764135088239397, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 7, "sum_logprob": -6.373825550079346, "mean_logprob": -0.9105465071541923, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 0.33631961307828395, "context_truncated": false, "elapsed_seconds": 0.027187195999431424 }, { "id": "code_completion-009", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def double_all(numbers):\n", "continuations": [ " return [n * 2 for n in numbers]\n", " return numbers * 0\n", " return [2 for n in numbers]\n", " return len(numbers) * 2\n" ], "tags": [ "python" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 15, "sum_logprob": -13.590858459472656, "mean_logprob": -0.9060572306315104, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 8, "sum_logprob": -21.125938415527344, "mean_logprob": -2.640742301940918, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 11, "sum_logprob": -25.994300842285156, "mean_logprob": -2.3631182583895596, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 11, "sum_logprob": -19.167556762695312, "mean_logprob": -1.7425051602450283, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.8364479296135179, "context_truncated": false, "elapsed_seconds": 0.04008716399948753 }, { "id": "code_completion-010", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def contains(items, value):\n", "continuations": [ " return items == value\n", " return value in items\n", " return items + value\n", " return items.index\n" ], "tags": [ "python" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -10.957561492919922, "mean_logprob": -1.565365927559989, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 6, "sum_logprob": -8.056070327758789, "mean_logprob": -1.342678387959798, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 7, "sum_logprob": -13.39298152923584, "mean_logprob": -1.91328307560512, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 6, "sum_logprob": -27.022537231445312, "mean_logprob": -4.503756205240886, "total_sequence_tokens": 14 } ], "best_vs_second_margin": 0.22268753960019083, "context_truncated": false, "elapsed_seconds": 0.02570848799950909 }, { "id": "code_completion-011", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def maximum(a, b):\n", "continuations": [ " return a + b\n", " return a < b\n", " return a if a > b else b\n", " return None\n" ], "tags": [ "python" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -12.686381340026855, "mean_logprob": -1.812340191432408, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 7, "sum_logprob": -12.447408676147461, "mean_logprob": -1.7782012394496374, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 11, "sum_logprob": -11.49631404876709, "mean_logprob": -1.0451194589788264, "total_sequence_tokens": 19 }, { "index": 3, "tokens": 4, "sum_logprob": -16.26229476928711, "mean_logprob": -4.065573692321777, "total_sequence_tokens": 12 } ], "best_vs_second_margin": 0.733081780470811, "context_truncated": false, "elapsed_seconds": 0.02868215599846735 }, { "id": "code_completion-012", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def repeat(text, count):\n", "continuations": [ " return count / text\n", " return len(text)\n", " return text + count\n", " return text * count\n" ], "tags": [ "python" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -24.464187622070312, "mean_logprob": -3.4948839460100447, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 7, "sum_logprob": -15.841794967651367, "mean_logprob": -2.263113566807338, "total_sequence_tokens": 15 }, { "index": 2, "tokens": 7, "sum_logprob": -15.606399536132812, "mean_logprob": -2.229485648018973, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 7, "sum_logprob": -11.067680358886719, "mean_logprob": -1.581097194126674, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 0.648388453892299, "context_truncated": false, "elapsed_seconds": 0.026035105998744257 }, { "id": "code_completion-013", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def keys_list(mapping):\n", "continuations": [ " return list(mapping.keys())\n", " return mapping.values\n", " return [mapping]\n", " return mapping.clear()\n" ], "tags": [ "python" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 9, "sum_logprob": -9.412752151489258, "mean_logprob": -1.045861350165473, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 6, "sum_logprob": -20.372913360595703, "mean_logprob": -3.3954855600992837, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 7, "sum_logprob": -17.14534568786621, "mean_logprob": -2.4493350982666016, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 7, "sum_logprob": -20.802125930786133, "mean_logprob": -2.9717322758265903, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 1.4034737481011286, "context_truncated": false, "elapsed_seconds": 0.028490945000157808 }, { "id": "code_completion-014", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def starts_with_a(text):\n", "continuations": [ " return text[-1] == \"a\"\n", " return text.startswith(\"a\")\n", " return \"a\" + text\n", " return text.count\n" ], "tags": [ "python" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 13, "sum_logprob": -20.738313674926758, "mean_logprob": -1.595254898071289, "total_sequence_tokens": 23 }, { "index": 1, "tokens": 11, "sum_logprob": -11.440147399902344, "mean_logprob": -1.040013399991122, "total_sequence_tokens": 21 }, { "index": 2, "tokens": 10, "sum_logprob": -15.75004768371582, "mean_logprob": -1.575004768371582, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 6, "sum_logprob": -20.53750991821289, "mean_logprob": -3.4229183197021484, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 0.5349913683804599, "context_truncated": false, "elapsed_seconds": 0.03216659999998228 }, { "id": "code_completion-015", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def total(numbers):\n", "continuations": [ " return len(numbers)\n", " return max(numbers)\n", " return sum(numbers)\n", " return numbers[0]\n" ], "tags": [ "python" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -10.798761367797852, "mean_logprob": -1.5426801953996931, "total_sequence_tokens": 13 }, { "index": 1, "tokens": 7, "sum_logprob": -11.562196731567383, "mean_logprob": -1.651742390223912, "total_sequence_tokens": 13 }, { "index": 2, "tokens": 7, "sum_logprob": -6.728759288787842, "mean_logprob": -0.9612513269696917, "total_sequence_tokens": 13 }, { "index": 3, "tokens": 7, "sum_logprob": -17.91793441772461, "mean_logprob": -2.559704916817801, "total_sequence_tokens": 13 } ], "best_vs_second_margin": 0.5814288684300014, "context_truncated": false, "elapsed_seconds": 0.02752010599942878 }, { "id": "code_completion-016", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 0.0, "item_elo": 950, "context": "def clamp_zero(x):\n", "continuations": [ " return min(0, x)\n", " return x == 0\n", " return -x\n", " return max(0, x)\n" ], "tags": [ "python" ], "prediction": 1, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 9, "sum_logprob": -17.8612060546875, "mean_logprob": -1.9845784505208333, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 8, "sum_logprob": -8.925451278686523, "mean_logprob": -1.1156814098358154, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 6, "sum_logprob": -12.237079620361328, "mean_logprob": -2.039513270060221, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 9, "sum_logprob": -11.028650283813477, "mean_logprob": -1.2254055870903864, "total_sequence_tokens": 17 } ], "best_vs_second_margin": 0.10972417725457095, "context_truncated": false, "elapsed_seconds": 0.02768399599881377 }, { "id": "code_completion-017", "category": "code_completion", "difficulty": "easy", "category_weight": 1.4, "difficulty_weight": 1.0, "point_weight": 1.4, "earned_points": 1.4, "item_elo": 950, "context": "def reversed_copy(items):\n", "continuations": [ " return items[::-1]\n", " return items[::1]\n", " return items.reverse\n", " return sorted(items)\n" ], "tags": [ "python" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -7.552888870239258, "mean_logprob": -0.9441111087799072, "total_sequence_tokens": 16 }, { "index": 1, "tokens": 8, "sum_logprob": -21.040647506713867, "mean_logprob": -2.6300809383392334, "total_sequence_tokens": 16 }, { "index": 2, "tokens": 6, "sum_logprob": -18.220312118530273, "mean_logprob": -3.0367186864217124, "total_sequence_tokens": 14 }, { "index": 3, "tokens": 7, "sum_logprob": -12.42491626739502, "mean_logprob": -1.7749880381992884, "total_sequence_tokens": 15 } ], "best_vs_second_margin": 0.8308769294193812, "context_truncated": false, "elapsed_seconds": 0.03015708699967945 }, { "id": "code_completion-018", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def count_positive(numbers):\n", "continuations": [ " return sum(numbers)\n", " return sum(1 for n in numbers if n > 0)\n", " return [n > 0 for n in numbers]\n", " return len(numbers) > 0\n" ], "tags": [ "python" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -12.093025207519531, "mean_logprob": -1.7275750296456474, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 17, "sum_logprob": -15.259834289550781, "mean_logprob": -0.897637311150046, "total_sequence_tokens": 25 }, { "index": 2, "tokens": 15, "sum_logprob": -15.651079177856445, "mean_logprob": -1.043405278523763, "total_sequence_tokens": 23 }, { "index": 3, "tokens": 11, "sum_logprob": -10.985816955566406, "mean_logprob": -0.9987106323242188, "total_sequence_tokens": 19 } ], "best_vs_second_margin": 0.10107332117417278, "context_truncated": false, "elapsed_seconds": 0.03531654699872888 }, { "id": "code_completion-019", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def unique_in_order(items):\n seen = set()\n", "continuations": [ " return set(items)\n", " return sorted(items)\n", " return [x for x in items if not (x in seen or seen.add(x))]\n", " return list(reversed(items))\n" ], "tags": [ "python" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -25.16358184814453, "mean_logprob": -3.5947974068777904, "total_sequence_tokens": 24 }, { "index": 1, "tokens": 7, "sum_logprob": -16.42459487915039, "mean_logprob": -2.3463706970214844, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 24, "sum_logprob": -38.50417709350586, "mean_logprob": -1.6043407122294109, "total_sequence_tokens": 41 }, { "index": 3, "tokens": 10, "sum_logprob": -23.19318389892578, "mean_logprob": -2.319318389892578, "total_sequence_tokens": 27 } ], "best_vs_second_margin": 0.7149776776631673, "context_truncated": false, "elapsed_seconds": 0.03663187399979506 }, { "id": "code_completion-020", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 0.0, "item_elo": 1050, "context": "def safe_divide(a, b):\n if b == 0:\n", "continuations": [ " return a\n return b / a\n", " b = 1\n return b\n", " raise StopIteration\n return a + b\n", " return None\n return a / b\n" ], "tags": [ "python" ], "prediction": 0, "label": 3, "passed": false, "choice_scores": [ { "index": 0, "tokens": 11, "sum_logprob": -14.361824989318848, "mean_logprob": -1.3056204535744407, "total_sequence_tokens": 30 }, { "index": 1, "tokens": 11, "sum_logprob": -31.49257469177246, "mean_logprob": -2.8629613356156782, "total_sequence_tokens": 30 }, { "index": 2, "tokens": 12, "sum_logprob": -27.718412399291992, "mean_logprob": -2.3098676999409995, "total_sequence_tokens": 31 }, { "index": 3, "tokens": 11, "sum_logprob": -19.33892250061035, "mean_logprob": -1.75808386369185, "total_sequence_tokens": 30 } ], "best_vs_second_margin": 0.4524634101174094, "context_truncated": false, "elapsed_seconds": 0.04077255999982299 }, { "id": "code_completion-021", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def word_lengths(words):\n", "continuations": [ " return {word: len(word) for word in words}\n", " return {len(words): word for word in words}\n", " return [words, len(words)]\n", " return words.keys()\n" ], "tags": [ "python" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 16, "sum_logprob": -14.82179069519043, "mean_logprob": -0.9263619184494019, "total_sequence_tokens": 25 }, { "index": 1, "tokens": 15, "sum_logprob": -34.53876495361328, "mean_logprob": -2.3025843302408853, "total_sequence_tokens": 24 }, { "index": 2, "tokens": 11, "sum_logprob": -27.680112838745117, "mean_logprob": -2.5163738944313745, "total_sequence_tokens": 20 }, { "index": 3, "tokens": 7, "sum_logprob": -24.124025344848633, "mean_logprob": -3.4462893349783763, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 1.3762224117914834, "context_truncated": false, "elapsed_seconds": 0.03476182600024913 }, { "id": "code_completion-022", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 0.0, "item_elo": 1050, "context": "def flatten_once(groups):\n", "continuations": [ " return list(groups)\n", " return [item for group in groups for item in group]\n", " return [groups[0]]\n", " return sum(len(g) for g in groups)\n" ], "tags": [ "python" ], "prediction": 3, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -14.239151954650879, "mean_logprob": -2.0341645649501254, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 15, "sum_logprob": -24.19076156616211, "mean_logprob": -1.6127174377441407, "total_sequence_tokens": 23 }, { "index": 2, "tokens": 9, "sum_logprob": -18.42530059814453, "mean_logprob": -2.047255622016059, "total_sequence_tokens": 17 }, { "index": 3, "tokens": 14, "sum_logprob": -21.162622451782227, "mean_logprob": -1.5116158894130163, "total_sequence_tokens": 22 } ], "best_vs_second_margin": 0.10110154833112439, "context_truncated": false, "elapsed_seconds": 0.03249907099962002 }, { "id": "code_completion-023", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def factorial(n):\n result = 1\n for value in range(2, n + 1):\n", "continuations": [ " result += value\n return n\n", " result = value\n return value\n", " result *= value\n return result\n", " n -= value\n return n\n" ], "tags": [ "python" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 10, "sum_logprob": -27.79298210144043, "mean_logprob": -2.779298210144043, "total_sequence_tokens": 38 }, { "index": 1, "tokens": 10, "sum_logprob": -37.84375762939453, "mean_logprob": -3.784375762939453, "total_sequence_tokens": 38 }, { "index": 2, "tokens": 10, "sum_logprob": -4.943729877471924, "mean_logprob": -0.4943729877471924, "total_sequence_tokens": 38 }, { "index": 3, "tokens": 10, "sum_logprob": -37.693763732910156, "mean_logprob": -3.7693763732910157, "total_sequence_tokens": 38 } ], "best_vs_second_margin": 2.2849252223968506, "context_truncated": false, "elapsed_seconds": 0.03755112500039104 }, { "id": "code_completion-024", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def first_index(items, target):\n for index, item in enumerate(items):\n if item == target:\n", "continuations": [ " continue\n return 0\n", " return item\n return target\n", " break\n return index\n", " return index\n return -1\n" ], "tags": [ "python" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -34.801002502441406, "mean_logprob": -4.350125312805176, "total_sequence_tokens": 37 }, { "index": 1, "tokens": 8, "sum_logprob": -26.48795509338379, "mean_logprob": -3.3109943866729736, "total_sequence_tokens": 37 }, { "index": 2, "tokens": 7, "sum_logprob": -13.000073432922363, "mean_logprob": -1.8571533475603377, "total_sequence_tokens": 36 }, { "index": 3, "tokens": 10, "sum_logprob": -8.2440185546875, "mean_logprob": -0.82440185546875, "total_sequence_tokens": 39 } ], "best_vs_second_margin": 1.0327514920915877, "context_truncated": false, "elapsed_seconds": 0.0376353020001261 }, { "id": "code_completion-025", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def merge_counts(left, right):\n result = dict(left)\n for key, value in right.items():\n", "continuations": [ " result[key] = result.get(key, 0) + value\n return result\n", " result[value] = key\n return left\n", " result.clear()\n return right\n", " key += value\n return result.values()\n" ], "tags": [ "python" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 24, "sum_logprob": -8.848027229309082, "mean_logprob": -0.36866780122121173, "total_sequence_tokens": 54 }, { "index": 1, "tokens": 13, "sum_logprob": -45.313819885253906, "mean_logprob": -3.4856784527118387, "total_sequence_tokens": 43 }, { "index": 2, "tokens": 10, "sum_logprob": -52.17894744873047, "mean_logprob": -5.217894744873047, "total_sequence_tokens": 40 }, { "index": 3, "tokens": 13, "sum_logprob": -37.48686599731445, "mean_logprob": -2.8836050767164965, "total_sequence_tokens": 43 } ], "best_vs_second_margin": 2.5149372754952846, "context_truncated": false, "elapsed_seconds": 0.048819237999850884 }, { "id": "code_completion-026", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def chunks(items, size):\n", "continuations": [ " return items[:size]\n", " return [items[i:i + size] for i in range(0, len(items), size)]\n", " return [size] * len(items)\n", " return items[::size][0]\n" ], "tags": [ "python" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -12.145590782165527, "mean_logprob": -1.735084397452218, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 27, "sum_logprob": -19.035940170288086, "mean_logprob": -0.705034821121781, "total_sequence_tokens": 35 }, { "index": 2, "tokens": 13, "sum_logprob": -21.432252883911133, "mean_logprob": -1.6486348372239332, "total_sequence_tokens": 21 }, { "index": 3, "tokens": 10, "sum_logprob": -35.489845275878906, "mean_logprob": -3.5489845275878906, "total_sequence_tokens": 18 } ], "best_vs_second_margin": 0.9436000161021523, "context_truncated": false, "elapsed_seconds": 0.03724283799965633 }, { "id": "code_completion-027", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def balanced_parentheses(text):\n depth = 0\n for char in text:\n if char == \"(\":\n depth += 1\n elif char == \")\":\n depth -= 1\n if depth < 0:\n return False\n", "continuations": [ " return True\n", " return depth > 0\n", " return depth == 0\n", " return len(text) == depth\n" ], "tags": [ "python" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -6.613285541534424, "mean_logprob": -1.653321385383606, "total_sequence_tokens": 72 }, { "index": 1, "tokens": 8, "sum_logprob": -13.789587020874023, "mean_logprob": -1.723698377609253, "total_sequence_tokens": 76 }, { "index": 2, "tokens": 8, "sum_logprob": -7.251691818237305, "mean_logprob": -0.9064614772796631, "total_sequence_tokens": 76 }, { "index": 3, "tokens": 10, "sum_logprob": -9.23131275177002, "mean_logprob": -0.923131275177002, "total_sequence_tokens": 78 } ], "best_vs_second_margin": 0.01666979789733891, "context_truncated": false, "elapsed_seconds": 0.06129010799850221 }, { "id": "code_completion-028", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def rotate_left(items, amount):\n if not items:\n return []\n", "continuations": [ " return items[::-1]\n", " return items[:amount]\n", " return items + amount\n", " amount %= len(items)\n return items[amount:] + items[:amount]\n" ], "tags": [ "python" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -15.63742446899414, "mean_logprob": -1.9546780586242676, "total_sequence_tokens": 29 }, { "index": 1, "tokens": 7, "sum_logprob": -11.773261070251465, "mean_logprob": -1.6818944386073522, "total_sequence_tokens": 28 }, { "index": 2, "tokens": 7, "sum_logprob": -18.23381996154785, "mean_logprob": -2.6048314230782643, "total_sequence_tokens": 28 }, { "index": 3, "tokens": 23, "sum_logprob": -23.40024757385254, "mean_logprob": -1.0174020684283713, "total_sequence_tokens": 44 } ], "best_vs_second_margin": 0.6644923701789809, "context_truncated": false, "elapsed_seconds": 0.05138291900038894 }, { "id": "code_completion-029", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def group_by_first_letter(words):\n groups = {}\n for word in words:\n", "continuations": [ " groups.setdefault(word[0], []).append(word)\n return groups\n", " groups[word] = word[0]\n return words\n", " groups.append(word[0])\n return groups\n", " word[0] = groups\n return groups.keys()\n" ], "tags": [ "python" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 21, "sum_logprob": -22.006641387939453, "mean_logprob": -1.047935304187593, "total_sequence_tokens": 47 }, { "index": 1, "tokens": 16, "sum_logprob": -43.108314514160156, "mean_logprob": -2.6942696571350098, "total_sequence_tokens": 42 }, { "index": 2, "tokens": 14, "sum_logprob": -30.153905868530273, "mean_logprob": -2.153850419180734, "total_sequence_tokens": 40 }, { "index": 3, "tokens": 16, "sum_logprob": -55.40138626098633, "mean_logprob": -3.4625866413116455, "total_sequence_tokens": 42 } ], "best_vs_second_margin": 1.105915114993141, "context_truncated": false, "elapsed_seconds": 0.03654749700035609 }, { "id": "code_completion-030", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def moving_pairs(items):\n", "continuations": [ " return zip(items, items)\n", " return list(zip(items, items[1:]))\n", " return items[::2]\n", " return [(items[0], items[-1])]\n" ], "tags": [ "python" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 9, "sum_logprob": -19.146240234375, "mean_logprob": -2.1273600260416665, "total_sequence_tokens": 17 }, { "index": 1, "tokens": 14, "sum_logprob": -15.492321968078613, "mean_logprob": -1.1065944262913294, "total_sequence_tokens": 22 }, { "index": 2, "tokens": 8, "sum_logprob": -18.950725555419922, "mean_logprob": -2.3688406944274902, "total_sequence_tokens": 16 }, { "index": 3, "tokens": 13, "sum_logprob": -26.311683654785156, "mean_logprob": -2.023975665752704, "total_sequence_tokens": 21 } ], "best_vs_second_margin": 0.9173812394613747, "context_truncated": false, "elapsed_seconds": 0.04552672099998745 }, { "id": "code_completion-031", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 0.0, "item_elo": 1050, "context": "def invert_mapping(mapping):\n", "continuations": [ " return dict(mapping)\n", " return list(mapping.items())\n", " return {value: key for key, value in mapping.items()}\n", " return mapping[::-1]\n" ], "tags": [ "python" ], "prediction": 3, "label": 2, "passed": false, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -16.197599411010742, "mean_logprob": -2.3139427730015347, "total_sequence_tokens": 15 }, { "index": 1, "tokens": 9, "sum_logprob": -14.858826637268066, "mean_logprob": -1.6509807374742296, "total_sequence_tokens": 17 }, { "index": 2, "tokens": 17, "sum_logprob": -18.217885971069336, "mean_logprob": -1.0716403512393726, "total_sequence_tokens": 25 }, { "index": 3, "tokens": 8, "sum_logprob": -8.22993278503418, "mean_logprob": -1.0287415981292725, "total_sequence_tokens": 16 } ], "best_vs_second_margin": 0.04289875311010016, "context_truncated": false, "elapsed_seconds": 0.04215201799888746 }, { "id": "code_completion-032", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def all_equal(items):\n iterator = iter(items)\n try:\n first = next(iterator)\n except StopIteration:\n", "continuations": [ " return False\n return first == iterator\n", " first = None\n return items == first\n", " return []\n return len(items)\n", " return True\n return all(item == first for item in iterator)\n" ], "tags": [ "python" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 11, "sum_logprob": -20.89051055908203, "mean_logprob": -1.8991373235529119, "total_sequence_tokens": 47 }, { "index": 1, "tokens": 13, "sum_logprob": -26.401443481445312, "mean_logprob": -2.0308802678034854, "total_sequence_tokens": 49 }, { "index": 2, "tokens": 12, "sum_logprob": -30.185457229614258, "mean_logprob": -2.5154547691345215, "total_sequence_tokens": 48 }, { "index": 3, "tokens": 18, "sum_logprob": -34.05173110961914, "mean_logprob": -1.8917628394232855, "total_sequence_tokens": 54 } ], "best_vs_second_margin": 0.00737448412962638, "context_truncated": false, "elapsed_seconds": 0.042933908000122756 }, { "id": "code_completion-033", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 0.0, "item_elo": 1050, "context": "def transpose(rows):\n", "continuations": [ " return [list(column) for column in zip(*rows)]\n", " return rows[::-1]\n", " return list(zip(rows))\n", " return [sum(row) for row in rows]\n" ], "tags": [ "python" ], "prediction": 3, "label": 0, "passed": false, "choice_scores": [ { "index": 0, "tokens": 16, "sum_logprob": -21.815868377685547, "mean_logprob": -1.3634917736053467, "total_sequence_tokens": 22 }, { "index": 1, "tokens": 8, "sum_logprob": -10.598787307739258, "mean_logprob": -1.3248484134674072, "total_sequence_tokens": 14 }, { "index": 2, "tokens": 9, "sum_logprob": -20.984830856323242, "mean_logprob": -2.3316478729248047, "total_sequence_tokens": 15 }, { "index": 3, "tokens": 14, "sum_logprob": -13.52786636352539, "mean_logprob": -0.9662761688232422, "total_sequence_tokens": 20 } ], "best_vs_second_margin": 0.35857224464416504, "context_truncated": false, "elapsed_seconds": 0.04276205599853711 }, { "id": "code_completion-034", "category": "code_completion", "difficulty": "medium", "category_weight": 1.4, "difficulty_weight": 1.5, "point_weight": 2.0999999999999996, "earned_points": 2.0999999999999996, "item_elo": 1050, "context": "def running_totals(numbers):\n totals = []\n current = 0\n for number in numbers:\n", "continuations": [ " totals.append(number)\n return current\n", " current += number\n totals.append(current)\n return totals\n", " current = number\n return [current]\n", " totals += current\n return numbers\n" ], "tags": [ "python" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 13, "sum_logprob": -39.600975036621094, "mean_logprob": -3.0462288489708533, "total_sequence_tokens": 44 }, { "index": 1, "tokens": 20, "sum_logprob": -6.991767883300781, "mean_logprob": -0.34958839416503906, "total_sequence_tokens": 51 }, { "index": 2, "tokens": 13, "sum_logprob": -40.40708923339844, "mean_logprob": -3.1082376333383412, "total_sequence_tokens": 44 }, { "index": 3, "tokens": 11, "sum_logprob": -54.728126525878906, "mean_logprob": -4.975284229625355, "total_sequence_tokens": 42 } ], "best_vs_second_margin": 2.6966404548058143, "context_truncated": false, "elapsed_seconds": 0.057458866000160924 }, { "id": "code_completion-035", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def binary_search(items, target):\n low, high = 0, len(items) - 1\n while low <= high:\n mid = (low + high) // 2\n if items[mid] == target:\n return mid\n if items[mid] < target:\n low = mid + 1\n else:\n", "continuations": [ " low = mid - 1\n return low\n", " high = mid + 1\n return high\n", " high = mid - 1\n return -1\n", " return mid\n return None\n" ], "tags": [ "python", "algorithm" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 14, "sum_logprob": -36.48191452026367, "mean_logprob": -2.605851037161691, "total_sequence_tokens": 106 }, { "index": 1, "tokens": 14, "sum_logprob": -46.85424041748047, "mean_logprob": -3.3467314583914622, "total_sequence_tokens": 106 }, { "index": 2, "tokens": 16, "sum_logprob": -7.23041296005249, "mean_logprob": -0.45190081000328064, "total_sequence_tokens": 108 }, { "index": 3, "tokens": 8, "sum_logprob": -23.18140983581543, "mean_logprob": -2.8976762294769287, "total_sequence_tokens": 100 } ], "best_vs_second_margin": 2.1539502271584103, "context_truncated": false, "elapsed_seconds": 0.07552583399956347 }, { "id": "code_completion-036", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def merge_sorted(left, right):\n result = []\n i = j = 0\n while i < len(left) and j < len(right):\n if left[i] <= right[j]:\n result.append(left[i])\n i += 1\n else:\n result.append(right[j])\n j += 1\n", "continuations": [ " return result\n", " result.extend(left)\n return result\n", " return sorted(left)\n", " result.extend(left[i:])\n result.extend(right[j:])\n return result\n" ], "tags": [ "python", "algorithm" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 4, "sum_logprob": -11.00175666809082, "mean_logprob": -2.750439167022705, "total_sequence_tokens": 100 }, { "index": 1, "tokens": 12, "sum_logprob": -27.22464370727539, "mean_logprob": -2.2687203089396157, "total_sequence_tokens": 108 }, { "index": 2, "tokens": 7, "sum_logprob": -48.96931457519531, "mean_logprob": -6.995616367885044, "total_sequence_tokens": 103 }, { "index": 3, "tokens": 24, "sum_logprob": -6.3913092613220215, "mean_logprob": -0.26630455255508423, "total_sequence_tokens": 120 } ], "best_vs_second_margin": 2.0024157563845315, "context_truncated": false, "elapsed_seconds": 0.07246739999936835 }, { "id": "code_completion-037", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def topological_visit(node, graph, visiting, visited, order):\n if node in visiting:\n raise ValueError(\"cycle\")\n if node in visited:\n return\n visiting.add(node)\n for neighbor in graph.get(node, []):\n topological_visit(neighbor, graph, visiting, visited, order)\n", "continuations": [ " visiting.remove(node)\n visited.add(node)\n order.append(node)\n", " order.append(node)\n visiting.add(node)\n", " visited.remove(node)\n order.clear()\n", " return graph[node]\n" ], "tags": [ "python", "graph" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 24, "sum_logprob": -22.875579833984375, "mean_logprob": -0.953149159749349, "total_sequence_tokens": 103 }, { "index": 1, "tokens": 16, "sum_logprob": -39.46855926513672, "mean_logprob": -2.466784954071045, "total_sequence_tokens": 95 }, { "index": 2, "tokens": 14, "sum_logprob": -21.016754150390625, "mean_logprob": -1.5011967250279017, "total_sequence_tokens": 93 }, { "index": 3, "tokens": 7, "sum_logprob": -17.03171157836914, "mean_logprob": -2.4331016540527344, "total_sequence_tokens": 86 } ], "best_vs_second_margin": 0.5480475652785527, "context_truncated": false, "elapsed_seconds": 0.07073609600047348 }, { "id": "code_completion-038", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 0.0, "item_elo": 1200, "context": "def longest_common_prefix(words):\n if not words:\n return \"\"\n prefix = words[0]\n for word in words[1:]:\n while not word.startswith(prefix):\n", "continuations": [ " prefix += word\n", " prefix = prefix[:-1]\n if not prefix:\n return \"\"\n return prefix\n", " word = word[:-1]\n return word\n", " return words[0]\n return prefix + word\n" ], "tags": [ "python", "string" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -3.5768799781799316, "mean_logprob": -0.5961466630299886, "total_sequence_tokens": 58 }, { "index": 1, "tokens": 25, "sum_logprob": -27.742603302001953, "mean_logprob": -1.1097041320800782, "total_sequence_tokens": 77 }, { "index": 2, "tokens": 14, "sum_logprob": -47.44060516357422, "mean_logprob": -3.3886146545410156, "total_sequence_tokens": 66 }, { "index": 3, "tokens": 14, "sum_logprob": -72.98765563964844, "mean_logprob": -5.213403974260602, "total_sequence_tokens": 66 } ], "best_vs_second_margin": 0.5135574690500896, "context_truncated": false, "elapsed_seconds": 0.06210656999974162 }, { "id": "code_completion-039", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def memoized_fibonacci(n, cache=None):\n if cache is None:\n cache = {0: 0, 1: 1}\n if n not in cache:\n", "continuations": [ " cache[n] = n\n", " return cache\n", " cache[n] = memoized_fibonacci(n - 1, cache) + memoized_fibonacci(n - 2, cache)\n return cache[n]\n", " cache.clear()\n return n\n" ], "tags": [ "python", "recursion" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 9, "sum_logprob": -11.39099407196045, "mean_logprob": -1.2656660079956055, "total_sequence_tokens": 56 }, { "index": 1, "tokens": 4, "sum_logprob": -30.46306037902832, "mean_logprob": -7.61576509475708, "total_sequence_tokens": 51 }, { "index": 2, "tokens": 45, "sum_logprob": -11.498723983764648, "mean_logprob": -0.2555271996392144, "total_sequence_tokens": 92 }, { "index": 3, "tokens": 10, "sum_logprob": -50.775535583496094, "mean_logprob": -5.077553558349609, "total_sequence_tokens": 57 } ], "best_vs_second_margin": 1.0101388083563911, "context_truncated": false, "elapsed_seconds": 0.0655426350003836 }, { "id": "code_completion-040", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def intervals_overlap(a_start, a_end, b_start, b_end):\n", "continuations": [ " return a_end < b_start\n", " return a_start == b_end\n", " return a_start + b_start\n", " return max(a_start, b_start) <= min(a_end, b_end)\n" ], "tags": [ "python", "interval" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 11, "sum_logprob": -16.90646743774414, "mean_logprob": -1.5369515852494673, "total_sequence_tokens": 33 }, { "index": 1, "tokens": 11, "sum_logprob": -21.989330291748047, "mean_logprob": -1.9990300265225498, "total_sequence_tokens": 33 }, { "index": 2, "tokens": 11, "sum_logprob": -18.211776733398438, "mean_logprob": -1.6556160666725852, "total_sequence_tokens": 33 }, { "index": 3, "tokens": 25, "sum_logprob": -22.111881256103516, "mean_logprob": -0.8844752502441406, "total_sequence_tokens": 47 } ], "best_vs_second_margin": 0.6524763350053266, "context_truncated": false, "elapsed_seconds": 0.04035365600066143 }, { "id": "code_completion-041", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def breadth_first_distances(graph, start):\n from collections import deque\n distances = {start: 0}\n queue = deque([start])\n while queue:\n node = queue.popleft()\n for neighbor in graph.get(node, []):\n if neighbor not in distances:\n", "continuations": [ " distances[neighbor] = distances[node] + 1\n queue.append(neighbor)\n return distances\n", " distances[node] += 1\n return queue\n", " queue.append(node)\n return graph\n", " return neighbor\n return distances[start]\n" ], "tags": [ "python", "graph" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 28, "sum_logprob": -6.422842025756836, "mean_logprob": -0.2293872152056013, "total_sequence_tokens": 107 }, { "index": 1, "tokens": 14, "sum_logprob": -47.448028564453125, "mean_logprob": -3.3891448974609375, "total_sequence_tokens": 93 }, { "index": 2, "tokens": 12, "sum_logprob": -68.20989990234375, "mean_logprob": -5.6841583251953125, "total_sequence_tokens": 91 }, { "index": 3, "tokens": 11, "sum_logprob": -64.69454956054688, "mean_logprob": -5.881322687322443, "total_sequence_tokens": 90 } ], "best_vs_second_margin": 3.159757682255336, "context_truncated": false, "elapsed_seconds": 0.06932938600039051 }, { "id": "code_completion-042", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def deep_get(mapping, path, default=None):\n current = mapping\n for key in path:\n if not isinstance(current, dict) or key not in current:\n return default\n", "continuations": [ " current = mapping[key]\n return mapping\n", " current = current[key]\n return current\n", " path = current\n return key\n", " return current.get(path)\n return default\n" ], "tags": [ "python", "nested_data" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 13, "sum_logprob": -32.41277313232422, "mean_logprob": -2.4932902409480167, "total_sequence_tokens": 61 }, { "index": 1, "tokens": 13, "sum_logprob": -4.214959621429443, "mean_logprob": -0.32422766318688023, "total_sequence_tokens": 61 }, { "index": 2, "tokens": 10, "sum_logprob": -39.24943923950195, "mean_logprob": -3.9249439239501953, "total_sequence_tokens": 58 }, { "index": 3, "tokens": 13, "sum_logprob": -58.77112579345703, "mean_logprob": -4.520855830265925, "total_sequence_tokens": 61 } ], "best_vs_second_margin": 2.1690625777611365, "context_truncated": false, "elapsed_seconds": 0.04932143499900121 }, { "id": "code_completion-043", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def stable_partition(items, predicate):\n matching = []\n other = []\n for item in items:\n if predicate(item):\n matching.append(item)\n else:\n other.append(item)\n", "continuations": [ " return sorted(items)\n", " return matching or other\n", " return matching + other\n", " return zip(matching, other)\n" ], "tags": [ "python", "algorithm" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 7, "sum_logprob": -30.302295684814453, "mean_logprob": -4.328899383544922, "total_sequence_tokens": 65 }, { "index": 1, "tokens": 6, "sum_logprob": -17.109447479248047, "mean_logprob": -2.8515745798746743, "total_sequence_tokens": 64 }, { "index": 2, "tokens": 7, "sum_logprob": -12.190431594848633, "mean_logprob": -1.7414902278355189, "total_sequence_tokens": 65 }, { "index": 3, "tokens": 9, "sum_logprob": -18.88239288330078, "mean_logprob": -2.0980436537000866, "total_sequence_tokens": 67 } ], "best_vs_second_margin": 0.35655342586456773, "context_truncated": false, "elapsed_seconds": 0.05759788499926799 }, { "id": "code_completion-044", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def parse_key_value(lines):\n result = {}\n for line in lines:\n line = line.strip()\n if not line or line.startswith(\"#\"):\n continue\n key, separator, value = line.partition(\"=\")\n if not separator:\n raise ValueError(\"missing equals sign\")\n", "continuations": [ " result[value] = separator\n return lines\n", " result[key] = key\n return value\n", " result.clear()\n return key\n", " result[key.strip()] = value.strip()\n return result\n" ], "tags": [ "python", "parsing" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 13, "sum_logprob": -49.231327056884766, "mean_logprob": -3.787025158221905, "total_sequence_tokens": 94 }, { "index": 1, "tokens": 13, "sum_logprob": -42.809051513671875, "mean_logprob": -3.293003962590144, "total_sequence_tokens": 94 }, { "index": 2, "tokens": 10, "sum_logprob": -46.267005920410156, "mean_logprob": -4.626700592041016, "total_sequence_tokens": 91 }, { "index": 3, "tokens": 18, "sum_logprob": -8.798966407775879, "mean_logprob": -0.48883146709865993, "total_sequence_tokens": 99 } ], "best_vs_second_margin": 2.804172495491484, "context_truncated": false, "elapsed_seconds": 0.07062074500026938 }, { "id": "code_completion-045", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def sliding_window_sums(numbers, width):\n if width <= 0 or width > len(numbers):\n return []\n current = sum(numbers[:width])\n result = [current]\n for index in range(width, len(numbers)):\n", "continuations": [ " current += numbers[index] - numbers[index - width]\n result.append(current)\n return result\n", " current = numbers[index]\n return [current]\n", " result.append(sum(numbers))\n return numbers\n", " width += index\n return result[:width]\n" ], "tags": [ "python", "algorithm" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 30, "sum_logprob": -41.30909729003906, "mean_logprob": -1.3769699096679688, "total_sequence_tokens": 98 }, { "index": 1, "tokens": 16, "sum_logprob": -62.10095977783203, "mean_logprob": -3.881309986114502, "total_sequence_tokens": 84 }, { "index": 2, "tokens": 14, "sum_logprob": -73.57785034179688, "mean_logprob": -5.255560738699777, "total_sequence_tokens": 82 }, { "index": 3, "tokens": 13, "sum_logprob": -64.48774719238281, "mean_logprob": -4.960595937875601, "total_sequence_tokens": 81 } ], "best_vs_second_margin": 2.504340076446533, "context_truncated": false, "elapsed_seconds": 0.07155449200035946 }, { "id": "code_completion-046", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 0.0, "item_elo": 1200, "context": "def retry(operation, attempts):\n last_error = None\n for _ in range(attempts):\n try:\n return operation()\n except Exception as error:\n last_error = error\n", "continuations": [ " return last_error\n", " if last_error is not None:\n raise last_error\n raise ValueError(\"attempts must be positive\")\n", " operation()\n return None\n", " except last_error:\n return attempts\n" ], "tags": [ "python", "exceptions" ], "prediction": 0, "label": 1, "passed": false, "choice_scores": [ { "index": 0, "tokens": 6, "sum_logprob": -7.404885768890381, "mean_logprob": -1.2341476281483967, "total_sequence_tokens": 57 }, { "index": 1, "tokens": 28, "sum_logprob": -61.165992736816406, "mean_logprob": -2.184499740600586, "total_sequence_tokens": 79 }, { "index": 2, "tokens": 8, "sum_logprob": -27.670310974121094, "mean_logprob": -3.4587888717651367, "total_sequence_tokens": 59 }, { "index": 3, "tokens": 11, "sum_logprob": -44.15677261352539, "mean_logprob": -4.014252055775035, "total_sequence_tokens": 62 } ], "best_vs_second_margin": 0.9503521124521892, "context_truncated": false, "elapsed_seconds": 0.05811959799939359 }, { "id": "code_completion-047", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def normalize_rows(matrix):\n result = []\n for row in matrix:\n total = sum(row)\n if total == 0:\n result.append([0.0 for _ in row])\n else:\n", "continuations": [ " result.append(row)\n", " result.append([total for _ in row])\n", " result.append([value / total for value in row])\n return result\n", " result = row / total\n return matrix\n" ], "tags": [ "python", "numeric" ], "prediction": 2, "label": 2, "passed": true, "choice_scores": [ { "index": 0, "tokens": 8, "sum_logprob": -12.316730499267578, "mean_logprob": -1.5395913124084473, "total_sequence_tokens": 67 }, { "index": 1, "tokens": 13, "sum_logprob": -21.836929321289062, "mean_logprob": -1.6797637939453125, "total_sequence_tokens": 72 }, { "index": 2, "tokens": 19, "sum_logprob": -18.89575958251953, "mean_logprob": -0.9945136622378701, "total_sequence_tokens": 78 }, { "index": 3, "tokens": 13, "sum_logprob": -36.02602005004883, "mean_logprob": -2.7712323115422177, "total_sequence_tokens": 72 } ], "best_vs_second_margin": 0.5450776501705772, "context_truncated": false, "elapsed_seconds": 0.04855151400079194 }, { "id": "code_completion-048", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def merge_ranges(ranges):\n if not ranges:\n return []\n ordered = sorted(ranges)\n merged = [list(ordered[0])]\n for start, end in ordered[1:]:\n previous = merged[-1]\n if start <= previous[1]:\n previous[1] = max(previous[1], end)\n else:\n", "continuations": [ " previous[0] = start\n return ranges\n", " merged = [[start, end]]\n return ordered\n", " return previous\n return merged\n", " merged.append([start, end])\n return [tuple(pair) for pair in merged]\n" ], "tags": [ "python", "interval" ], "prediction": 3, "label": 3, "passed": true, "choice_scores": [ { "index": 0, "tokens": 13, "sum_logprob": -77.74765014648438, "mean_logprob": -5.98058847280649, "total_sequence_tokens": 105 }, { "index": 1, "tokens": 15, "sum_logprob": -66.21233367919922, "mean_logprob": -4.414155578613281, "total_sequence_tokens": 107 }, { "index": 2, "tokens": 8, "sum_logprob": -38.67328643798828, "mean_logprob": -4.834160804748535, "total_sequence_tokens": 100 }, { "index": 3, "tokens": 24, "sum_logprob": -42.53789520263672, "mean_logprob": -1.7724123001098633, "total_sequence_tokens": 116 } ], "best_vs_second_margin": 2.6417432785034176, "context_truncated": false, "elapsed_seconds": 0.07347782599936181 }, { "id": "code_completion-049", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def count_paths(grid, row=0, col=0, memo=None):\n if memo is None:\n memo = {}\n if row >= len(grid) or col >= len(grid[0]) or grid[row][col] == 1:\n return 0\n if (row, col) == (len(grid) - 1, len(grid[0]) - 1):\n return 1\n if (row, col) not in memo:\n", "continuations": [ " memo[(row, col)] = count_paths(grid, row + 1, col, memo) + count_paths(grid, row, col + 1, memo)\n return memo[(row, col)]\n", " memo.clear()\n return 0\n", " return count_paths(grid, row - 1, col - 1, memo)\n return memo\n", " memo[(row, col)] = 1\n return len(memo)\n" ], "tags": [ "python", "dynamic_programming" ], "prediction": 0, "label": 0, "passed": true, "choice_scores": [ { "index": 0, "tokens": 53, "sum_logprob": -6.174004554748535, "mean_logprob": -0.11649065197638746, "total_sequence_tokens": 175 }, { "index": 1, "tokens": 11, "sum_logprob": -51.11182403564453, "mean_logprob": -4.646529457785866, "total_sequence_tokens": 133 }, { "index": 2, "tokens": 27, "sum_logprob": -71.41888427734375, "mean_logprob": -2.6451438621238426, "total_sequence_tokens": 149 }, { "index": 3, "tokens": 20, "sum_logprob": -37.429500579833984, "mean_logprob": -1.8714750289916993, "total_sequence_tokens": 142 } ], "best_vs_second_margin": 1.7549843770153117, "context_truncated": false, "elapsed_seconds": 0.10541650500090327 }, { "id": "code_completion-050", "category": "code_completion", "difficulty": "hard", "category_weight": 1.4, "difficulty_weight": 2.25, "point_weight": 3.15, "earned_points": 3.15, "item_elo": 1200, "context": "def compose(*functions):\n def composed(value):\n result = value\n for function in reversed(functions):\n", "continuations": [ " result = functions(result)\n return value\n", " result = function(result)\n return result\n return composed\n", " return function\n return functions\n return result\n", " result.append(function)\n return result\n return functions\n" ], "tags": [ "python", "higher_order" ], "prediction": 1, "label": 1, "passed": true, "choice_scores": [ { "index": 0, "tokens": 13, "sum_logprob": -36.70912170410156, "mean_logprob": -2.823778592623197, "total_sequence_tokens": 41 }, { "index": 1, "tokens": 17, "sum_logprob": -9.565998077392578, "mean_logprob": -0.562705769258387, "total_sequence_tokens": 45 }, { "index": 2, "tokens": 12, "sum_logprob": -66.07337951660156, "mean_logprob": -5.506114959716797, "total_sequence_tokens": 40 }, { "index": 3, "tokens": 16, "sum_logprob": -38.47551727294922, "mean_logprob": -2.404719829559326, "total_sequence_tokens": 44 } ], "best_vs_second_margin": 1.8420140603009392, "context_truncated": false, "elapsed_seconds": 0.0417035800001031 } ] }