19764 lines
541 KiB
JSON
19764 lines
541 KiB
JSON
{
|
|
"created_at": "2026-07-26T17:00:32.578538+00:00",
|
|
"benchmark": "BananaMind Base Bench 1.1",
|
|
"benchmark_version": "1.1",
|
|
"dataset_id": "BananaMind/BananaMind-Base-Bench-1.1",
|
|
"dataset_revision": "main",
|
|
"dataset_file": "/home/appvoid/.cache/huggingface/hub/datasets--BananaMind--BananaMind-Base-Bench-1.1/snapshots/266ef5dc7c812b83c5205235f966ae2f41b3e5b1/data/test.jsonl",
|
|
"dataset_sha256": "2f563bb46df778ca494fa20f994a8d3045d4c51fbbffeee433764e2813abea21",
|
|
"model": "/home/appvoid/palmer-006",
|
|
"model_revision": "main",
|
|
"tokenizer": "/home/appvoid/palmer-006",
|
|
"device": "cuda",
|
|
"dtype": "bfloat16",
|
|
"model_context_length": 262144,
|
|
"scoring": {
|
|
"task": "four-choice base-text continuation",
|
|
"choice_score": "mean conditional token log-probability",
|
|
"chat_template": false,
|
|
"generation": false,
|
|
"add_special_tokens": false,
|
|
"add_bos": false
|
|
},
|
|
"elo": {
|
|
"scale": 400.0,
|
|
"prior_rating": 1000.0,
|
|
"prior_weight": 4.0,
|
|
"method": "weighted fixed-item logistic maximum-likelihood rating"
|
|
},
|
|
"summary": {
|
|
"official_complete_run": true,
|
|
"cases": 350,
|
|
"passed": 231,
|
|
"accuracy": 0.66,
|
|
"weighted_points": 425.525,
|
|
"possible_weighted_points": 668.1875,
|
|
"weighted_accuracy": 0.6368347207931905,
|
|
"overall_elo": 1124,
|
|
"overall_elo_unrounded": 1123.965054004459,
|
|
"categories": {
|
|
"language_completion": {
|
|
"cases": 50,
|
|
"passed": 47,
|
|
"accuracy": 0.94,
|
|
"weighted_points": 73.75,
|
|
"possible_weighted_points": 78.5,
|
|
"weighted_accuracy": 0.9394904458598726,
|
|
"elo": 1345,
|
|
"elo_unrounded": 1345.2360136505486
|
|
},
|
|
"commonsense": {
|
|
"cases": 50,
|
|
"passed": 35,
|
|
"accuracy": 0.7,
|
|
"weighted_points": 58.85,
|
|
"possible_weighted_points": 87.17500000000001,
|
|
"weighted_accuracy": 0.6750788643533122,
|
|
"elo": 1084,
|
|
"elo_unrounded": 1084.314352251316
|
|
},
|
|
"world_knowledge": {
|
|
"cases": 50,
|
|
"passed": 38,
|
|
"accuracy": 0.76,
|
|
"weighted_points": 66.0,
|
|
"possible_weighted_points": 87.72500000000001,
|
|
"weighted_accuracy": 0.7523510971786833,
|
|
"elo": 1151,
|
|
"elo_unrounded": 1150.5543694613593
|
|
},
|
|
"context_tracking": {
|
|
"cases": 50,
|
|
"passed": 26,
|
|
"accuracy": 0.52,
|
|
"weighted_points": 47.699999999999996,
|
|
"possible_weighted_points": 94.19999999999999,
|
|
"weighted_accuracy": 0.5063694267515924,
|
|
"elo": 1002,
|
|
"elo_unrounded": 1002.410277417023
|
|
},
|
|
"quantitative": {
|
|
"cases": 50,
|
|
"passed": 17,
|
|
"accuracy": 0.34,
|
|
"weighted_points": 34.125,
|
|
"possible_weighted_points": 103.025,
|
|
"weighted_accuracy": 0.3312302839116719,
|
|
"elo": 923,
|
|
"elo_unrounded": 922.9160309715924
|
|
},
|
|
"logical_reasoning": {
|
|
"cases": 50,
|
|
"passed": 25,
|
|
"accuracy": 0.5,
|
|
"weighted_points": 51.300000000000004,
|
|
"possible_weighted_points": 107.66250000000001,
|
|
"weighted_accuracy": 0.47648902821316613,
|
|
"elo": 1082,
|
|
"elo_unrounded": 1081.6919799942607
|
|
},
|
|
"code_completion": {
|
|
"cases": 50,
|
|
"passed": 43,
|
|
"accuracy": 0.86,
|
|
"weighted_points": 93.8,
|
|
"possible_weighted_points": 109.89999999999999,
|
|
"weighted_accuracy": 0.8535031847133758,
|
|
"elo": 1403,
|
|
"elo_unrounded": 1402.753907086344
|
|
}
|
|
},
|
|
"difficulties": {
|
|
"easy": {
|
|
"cases": 117,
|
|
"passed": 83,
|
|
"accuracy": 0.7094017094017094,
|
|
"weighted_points": 98.60000000000001,
|
|
"possible_weighted_points": 141.2,
|
|
"weighted_accuracy": 0.6983002832861192,
|
|
"elo": 1016,
|
|
"elo_unrounded": 1016.3133811577516
|
|
},
|
|
"medium": {
|
|
"cases": 117,
|
|
"passed": 77,
|
|
"accuracy": 0.6581196581196581,
|
|
"weighted_points": 137.925,
|
|
"possible_weighted_points": 211.875,
|
|
"weighted_accuracy": 0.6509734513274337,
|
|
"elo": 1077,
|
|
"elo_unrounded": 1076.9044441063706
|
|
},
|
|
"hard": {
|
|
"cases": 116,
|
|
"passed": 71,
|
|
"accuracy": 0.6120689655172413,
|
|
"weighted_points": 189.0,
|
|
"possible_weighted_points": 315.1125,
|
|
"weighted_accuracy": 0.599785790789004,
|
|
"elo": 1187,
|
|
"elo_unrounded": 1186.5309969349169
|
|
}
|
|
},
|
|
"context_truncations": 0
|
|
},
|
|
"results": [
|
|
{
|
|
"id": "language_completion-001",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "Dark clouds gathered above the park, so Mira opened",
|
|
"continuations": [
|
|
" a sandwich loudly.",
|
|
" the sunlight.",
|
|
" her umbrella.",
|
|
" the calendar yesterday."
|
|
],
|
|
"tags": [
|
|
"coherence"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -29.258682250976562,
|
|
"mean_logprob": -4.876447041829427,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.473627090454102,
|
|
"mean_logprob": -4.8245423634847,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -8.218729019165039,
|
|
"mean_logprob": -1.3697881698608398,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -22.170177459716797,
|
|
"mean_logprob": -5.542544364929199,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.4547541936238604,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.9227663130004657
|
|
},
|
|
{
|
|
"id": "language_completion-002",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "The librarian asked everyone to speak",
|
|
"continuations": [
|
|
" brightly.",
|
|
" quietly.",
|
|
" woodenly.",
|
|
" tomorrow."
|
|
],
|
|
"tags": [
|
|
"adverb"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.653277397155762,
|
|
"mean_logprob": -5.217759132385254,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.187465190887451,
|
|
"mean_logprob": -2.395821730295817,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -20.17020034790039,
|
|
"mean_logprob": -6.723400115966797,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.879611015319824,
|
|
"mean_logprob": -5.939805507659912,
|
|
"total_sequence_tokens": 10
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.821937402089437,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.051079696999295265
|
|
},
|
|
{
|
|
"id": "language_completion-003",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 0.0,
|
|
"item_elo": 750,
|
|
"context": "The brass key fit perfectly into the",
|
|
"continuations": [
|
|
" pillow.",
|
|
" thunder.",
|
|
" lock.",
|
|
" soup."
|
|
],
|
|
"tags": [
|
|
"collocation"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.946988105773926,
|
|
"mean_logprob": -3.648996035257975,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -18.382553100585938,
|
|
"mean_logprob": -6.1275177001953125,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.96212100982666,
|
|
"mean_logprob": -3.98106050491333,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -14.014673233032227,
|
|
"mean_logprob": -7.007336616516113,
|
|
"total_sequence_tokens": 10
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.33206446965535497,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.043018684000344365
|
|
},
|
|
{
|
|
"id": "language_completion-004",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "After kneading the dough, the baker placed it in the",
|
|
"continuations": [
|
|
" envelope.",
|
|
" freezer to read.",
|
|
" bicycle.",
|
|
" oven."
|
|
],
|
|
"tags": [
|
|
"coherence"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -16.552427291870117,
|
|
"mean_logprob": -8.276213645935059,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -25.37650489807129,
|
|
"mean_logprob": -5.075300979614258,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -25.005922317504883,
|
|
"mean_logprob": -12.502961158752441,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -2.8407983779907227,
|
|
"mean_logprob": -1.4203991889953613,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.6549017906188963,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05098429600002419
|
|
},
|
|
{
|
|
"id": "language_completion-005",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "Leo was exhausted after the hike and went straight to",
|
|
"continuations": [
|
|
" bed.",
|
|
" the trumpet.",
|
|
" blue.",
|
|
" calculate."
|
|
],
|
|
"tags": [
|
|
"coherence"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -2.9319279193878174,
|
|
"mean_logprob": -1.4659639596939087,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -15.567556381225586,
|
|
"mean_logprob": -3.1135112762451174,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.717717170715332,
|
|
"mean_logprob": -5.358858585357666,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -17.180051803588867,
|
|
"mean_logprob": -8.590025901794434,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.6475473165512087,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.041541919999872334
|
|
},
|
|
{
|
|
"id": "language_completion-006",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "The ice cube disappeared because it had",
|
|
"continuations": [
|
|
" flown north.",
|
|
" melted.",
|
|
" written a letter.",
|
|
" become louder."
|
|
],
|
|
"tags": [
|
|
"verb"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.799678802490234,
|
|
"mean_logprob": -3.9499197006225586,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.6080100536346436,
|
|
"mean_logprob": -1.2026700178782146,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.65917205810547,
|
|
"mean_logprob": -4.414793014526367,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -17.602617263793945,
|
|
"mean_logprob": -3.520523452758789,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.3178534348805746,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0459696409998287
|
|
},
|
|
{
|
|
"id": "language_completion-007",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "One child waited outside; soon, three more",
|
|
"continuations": [
|
|
" childs arrived.",
|
|
" childes arrived.",
|
|
" children arrived.",
|
|
" child arriveds."
|
|
],
|
|
"tags": [
|
|
"grammar"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.999372482299805,
|
|
"mean_logprob": -3.749843120574951,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -24.83102798461914,
|
|
"mean_logprob": -6.207756996154785,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.094472885131836,
|
|
"mean_logprob": -2.031490961710612,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -26.04423713684082,
|
|
"mean_logprob": -6.511059284210205,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.7183521588643393,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04569892300060019
|
|
},
|
|
{
|
|
"id": "language_completion-008",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "Yesterday, Priya walked to the station and",
|
|
"continuations": [
|
|
" waits for tomorrow.",
|
|
" waiting there now.",
|
|
" will walked home.",
|
|
" bought a ticket."
|
|
],
|
|
"tags": [
|
|
"tense"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -27.789226531982422,
|
|
"mean_logprob": -5.5578453063964846,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -24.613021850585938,
|
|
"mean_logprob": -6.153255462646484,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -27.470468521118164,
|
|
"mean_logprob": -6.867617130279541,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.572569847106934,
|
|
"mean_logprob": -2.3931424617767334,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.164702844619751,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.043610005001028185
|
|
},
|
|
{
|
|
"id": "language_completion-009",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "The cup is empty, but the pitcher is",
|
|
"continuations": [
|
|
" full.",
|
|
" emptier than silence.",
|
|
" running yesterday.",
|
|
" carefully."
|
|
],
|
|
"tags": [
|
|
"contrast"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -0.5352787971496582,
|
|
"mean_logprob": -0.2676393985748291,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -30.967300415039062,
|
|
"mean_logprob": -5.161216735839844,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -23.753543853759766,
|
|
"mean_logprob": -7.917847951253255,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -22.881458282470703,
|
|
"mean_logprob": -11.440729141235352,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.893577337265015,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04228099199826829
|
|
},
|
|
{
|
|
"id": "language_completion-010",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "Neither the teacher nor the students",
|
|
"continuations": [
|
|
" was arrives.",
|
|
" were ready.",
|
|
" is readiness.",
|
|
" be readying."
|
|
],
|
|
"tags": [
|
|
"agreement"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -25.68792152404785,
|
|
"mean_logprob": -6.421980381011963,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.509971618652344,
|
|
"mean_logprob": -3.1699905395507812,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -28.661731719970703,
|
|
"mean_logprob": -7.165432929992676,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -28.500198364257812,
|
|
"mean_logprob": -7.125049591064453,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.2519898414611816,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05342107900105475
|
|
},
|
|
{
|
|
"id": "language_completion-011",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "Nora packed an apple, a bottle of water, and",
|
|
"continuations": [
|
|
" because.",
|
|
" beneath.",
|
|
" a sandwich.",
|
|
" although."
|
|
],
|
|
"tags": [
|
|
"parallelism"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -21.972896575927734,
|
|
"mean_logprob": -10.986448287963867,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -21.476001739501953,
|
|
"mean_logprob": -10.738000869750977,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.899407386779785,
|
|
"mean_logprob": -1.9748518466949463,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -24.667869567871094,
|
|
"mean_logprob": -12.333934783935547,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 8.76314902305603,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03638890000001993
|
|
},
|
|
{
|
|
"id": "language_completion-012",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "A robin is a bird, and a salmon is",
|
|
"continuations": [
|
|
" beneath flying.",
|
|
" an quickly.",
|
|
" yesterday's wing.",
|
|
" a fish."
|
|
],
|
|
"tags": [
|
|
"syntax"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -33.78597640991211,
|
|
"mean_logprob": -11.26199213663737,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -33.265933990478516,
|
|
"mean_logprob": -11.088644663492838,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -36.72492980957031,
|
|
"mean_logprob": -7.3449859619140625,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -0.36596569418907166,
|
|
"mean_logprob": -0.12198856472969055,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 7.222997397184372,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03459843299970089
|
|
},
|
|
{
|
|
"id": "language_completion-013",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "She adopted a puppy from",
|
|
"continuations": [
|
|
" an animal shelter.",
|
|
" shelter animal an.",
|
|
" a sheltering quickly.",
|
|
" the adopted because."
|
|
],
|
|
"tags": [
|
|
"article"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.865048408508301,
|
|
"mean_logprob": -1.9662621021270752,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -32.348663330078125,
|
|
"mean_logprob": -8.087165832519531,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -33.793785095214844,
|
|
"mean_logprob": -6.758757019042969,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -30.407835006713867,
|
|
"mean_logprob": -7.601958751678467,
|
|
"total_sequence_tokens": 10
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.792494916915894,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03482184299900837
|
|
},
|
|
{
|
|
"id": "language_completion-014",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "Daniel thanked Rosa because",
|
|
"continuations": [
|
|
" him had helped she.",
|
|
" she had helped him.",
|
|
" her helped himself.",
|
|
" they helps he."
|
|
],
|
|
"tags": [
|
|
"pronouns"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -32.18361282348633,
|
|
"mean_logprob": -6.4367225646972654,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -11.391644477844238,
|
|
"mean_logprob": -2.2783288955688477,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -28.635631561279297,
|
|
"mean_logprob": -7.158907890319824,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -32.779876708984375,
|
|
"mean_logprob": -8.194969177246094,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.158393669128418,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03904251700078021
|
|
},
|
|
{
|
|
"id": "language_completion-015",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "This suitcase is heavy, but that one is even",
|
|
"continuations": [
|
|
" most heavy.",
|
|
" heavily one.",
|
|
" heavier.",
|
|
" more heavier."
|
|
],
|
|
"tags": [
|
|
"comparative"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.571857452392578,
|
|
"mean_logprob": -5.523952484130859,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -23.978748321533203,
|
|
"mean_logprob": -7.992916107177734,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -1.3102918863296509,
|
|
"mean_logprob": -0.4367639621098836,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.343090057373047,
|
|
"mean_logprob": -2.0857725143432617,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.6490085522333782,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03316458200060879
|
|
},
|
|
{
|
|
"id": "language_completion-016",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "Monday comes immediately after",
|
|
"continuations": [
|
|
" Wednesday.",
|
|
" evening.",
|
|
" January.",
|
|
" Sunday."
|
|
],
|
|
"tags": [
|
|
"sequence"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.539777755737305,
|
|
"mean_logprob": -4.769888877868652,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.153751373291016,
|
|
"mean_logprob": -5.576875686645508,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.202709197998047,
|
|
"mean_logprob": -5.601354598999023,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.936887264251709,
|
|
"mean_logprob": -3.9684436321258545,
|
|
"total_sequence_tokens": 7
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8014452457427979,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04127686799984076
|
|
},
|
|
{
|
|
"id": "language_completion-017",
|
|
"category": "language_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.0,
|
|
"earned_points": 1.0,
|
|
"item_elo": 750,
|
|
"context": "Although the road was narrow, the driver remained",
|
|
"continuations": [
|
|
" careful.",
|
|
" width.",
|
|
" carefully road.",
|
|
" narrowness."
|
|
],
|
|
"tags": [
|
|
"grammar"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.293829917907715,
|
|
"mean_logprob": -3.6469149589538574,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -20.16122055053711,
|
|
"mean_logprob": -10.080610275268555,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -23.18958854675293,
|
|
"mean_logprob": -7.7298628489176435,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -20.18323516845703,
|
|
"mean_logprob": -6.727745056152344,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.0808300971984863,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.033489133000330185
|
|
},
|
|
{
|
|
"id": "language_completion-018",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 0.0,
|
|
"item_elo": 850,
|
|
"context": "The forecast promised sunshine. Nevertheless, we packed",
|
|
"continuations": [
|
|
" the forecast itself.",
|
|
" rain jackets.",
|
|
" a darker noon.",
|
|
" sunshine into boxes."
|
|
],
|
|
"tags": [
|
|
"discourse"
|
|
],
|
|
"prediction": 3,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -19.17872428894043,
|
|
"mean_logprob": -4.794681072235107,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.765783309936523,
|
|
"mean_logprob": -4.191445827484131,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -29.223602294921875,
|
|
"mean_logprob": -4.8706003824869795,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -22.123672485351562,
|
|
"mean_logprob": -3.6872787475585938,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5041670799255371,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03606204700008675
|
|
},
|
|
{
|
|
"id": "language_completion-019",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "Amir revised the report twice before submitting it, thereby",
|
|
"continuations": [
|
|
" forgetting what reports are.",
|
|
" submitting before revising.",
|
|
" reducing the chance of errors.",
|
|
" making twice become three."
|
|
],
|
|
"tags": [
|
|
"discourse"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -40.61179733276367,
|
|
"mean_logprob": -6.768632888793945,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -27.6369686126709,
|
|
"mean_logprob": -4.606161435445149,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -12.523467063903809,
|
|
"mean_logprob": -2.0872445106506348,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -44.723777770996094,
|
|
"mean_logprob": -8.944755554199219,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.5189169247945147,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.034903477999250754
|
|
},
|
|
{
|
|
"id": "language_completion-020",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "The museum closes at six. We arrived at five forty-five, leaving us",
|
|
"continuations": [
|
|
" closed before arrival.",
|
|
" six museums to visit.",
|
|
" yesterday instead.",
|
|
" only fifteen minutes."
|
|
],
|
|
"tags": [
|
|
"coherence"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -27.271808624267578,
|
|
"mean_logprob": -6.8179521560668945,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -19.100353240966797,
|
|
"mean_logprob": -3.8200706481933593,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -22.578886032104492,
|
|
"mean_logprob": -7.526295344034831,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.414216995239258,
|
|
"mean_logprob": -3.3535542488098145,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.46651639938354483,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03929743299886468
|
|
},
|
|
{
|
|
"id": "language_completion-021",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "The committee postponed the vote until every member had",
|
|
"continuations": [
|
|
" reviewed the proposal.",
|
|
" been proposal by voting.",
|
|
" postponed every committee.",
|
|
" review the voted."
|
|
],
|
|
"tags": [
|
|
"aspect"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.851517677307129,
|
|
"mean_logprob": -2.7128794193267822,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -34.83161544799805,
|
|
"mean_logprob": -6.9663230895996096,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -32.971126556396484,
|
|
"mean_logprob": -5.495187759399414,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -32.07133483886719,
|
|
"mean_logprob": -8.017833709716797,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.782308340072632,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.030963249000706128
|
|
},
|
|
{
|
|
"id": "language_completion-022",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "Had Elena known the bridge was closed, she",
|
|
"continuations": [
|
|
" takes that route anyway.",
|
|
" would have taken another route.",
|
|
" will know it tomorrow yesterday.",
|
|
" had closes the bridge."
|
|
],
|
|
"tags": [
|
|
"conditional"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -27.9699649810791,
|
|
"mean_logprob": -5.59399299621582,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -18.3637752532959,
|
|
"mean_logprob": -3.06062920888265,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -39.02067184448242,
|
|
"mean_logprob": -6.503445307413737,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -29.636816024780273,
|
|
"mean_logprob": -4.939469337463379,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.878840128580729,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03254597900013323
|
|
},
|
|
{
|
|
"id": "language_completion-023",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "The novel was praised not only for its plot but also for",
|
|
"continuations": [
|
|
" it was long.",
|
|
" plotting beyond.",
|
|
" its vivid characters.",
|
|
" praise was also."
|
|
],
|
|
"tags": [
|
|
"parallelism"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -33.65026092529297,
|
|
"mean_logprob": -8.412565231323242,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -31.721717834472656,
|
|
"mean_logprob": -7.930429458618164,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.388650894165039,
|
|
"mean_logprob": -3.3471627235412598,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -50.499691009521484,
|
|
"mean_logprob": -10.099938201904298,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.583266735076904,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0373120789990935
|
|
},
|
|
{
|
|
"id": "language_completion-024",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "Because the instructions were ambiguous, two technicians interpreted them",
|
|
"continuations": [
|
|
" with identical ambiguity only.",
|
|
" before they were written.",
|
|
" into a screwdriver.",
|
|
" differently."
|
|
],
|
|
"tags": [
|
|
"semantics"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -30.603696823120117,
|
|
"mean_logprob": -6.120739364624024,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -17.289165496826172,
|
|
"mean_logprob": -3.457833099365234,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -19.69405174255371,
|
|
"mean_logprob": -3.282341957092285,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -2.138988971710205,
|
|
"mean_logprob": -1.0694944858551025,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.2128474712371826,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02990865999890957
|
|
},
|
|
{
|
|
"id": "language_completion-025",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "Sofia lowered her voice so that she would not",
|
|
"continuations": [
|
|
" wake the baby.",
|
|
" sleeping the quiet.",
|
|
" lower the voice's color.",
|
|
" have loudly whisper."
|
|
],
|
|
"tags": [
|
|
"purpose"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.679588317871094,
|
|
"mean_logprob": -4.169897079467773,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -38.57061767578125,
|
|
"mean_logprob": -9.642654418945312,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -33.39238739013672,
|
|
"mean_logprob": -4.770341055733817,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -32.384063720703125,
|
|
"mean_logprob": -5.3973439534505205,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6004439762660434,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03863776300022437
|
|
},
|
|
{
|
|
"id": "language_completion-026",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "The first experiment failed; the second, by contrast,",
|
|
"continuations": [
|
|
" was experiment first.",
|
|
" produced reliable results.",
|
|
" contrasted before being second.",
|
|
" failing reliably."
|
|
],
|
|
"tags": [
|
|
"discourse"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -27.998268127441406,
|
|
"mean_logprob": -6.999567031860352,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.488460540771484,
|
|
"mean_logprob": -2.372115135192871,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -40.71430206298828,
|
|
"mean_logprob": -6.785717010498047,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -21.225833892822266,
|
|
"mean_logprob": -5.306458473205566,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.9343433380126953,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028666005999184563
|
|
},
|
|
{
|
|
"id": "language_completion-027",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "Every applicant must attach a resume, regardless of",
|
|
"continuations": [
|
|
" attaching is a verb.",
|
|
" resumes being paper.",
|
|
" prior experience.",
|
|
" every regardless."
|
|
],
|
|
"tags": [
|
|
"collocation"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -33.530799865722656,
|
|
"mean_logprob": -5.588466644287109,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -28.748760223388672,
|
|
"mean_logprob": -5.749752044677734,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.654231071472168,
|
|
"mean_logprob": -3.218077023824056,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -31.807788848876953,
|
|
"mean_logprob": -10.602596282958984,
|
|
"total_sequence_tokens": 13
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.3703896204630532,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02823326300131157
|
|
},
|
|
{
|
|
"id": "language_completion-028",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "The recipe calls for fresh basil; dried basil may be used",
|
|
"continuations": [
|
|
" before fresh basil exists.",
|
|
" to call the recipe.",
|
|
" as a plate.",
|
|
" as a substitute."
|
|
],
|
|
"tags": [
|
|
"coherence"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -19.658309936523438,
|
|
"mean_logprob": -3.2763849894205728,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -22.95904541015625,
|
|
"mean_logprob": -4.59180908203125,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -20.08816146850586,
|
|
"mean_logprob": -5.022040367126465,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.103679180145264,
|
|
"mean_logprob": -1.025919795036316,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.250465194384257,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03386163600043801
|
|
},
|
|
{
|
|
"id": "language_completion-029",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "Far from being discouraged by the criticism, Mei became",
|
|
"continuations": [
|
|
" more determined.",
|
|
" criticism itself.",
|
|
" far from becoming.",
|
|
" determine yesterday."
|
|
],
|
|
"tags": [
|
|
"idiom"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.815675735473633,
|
|
"mean_logprob": -2.938558578491211,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -23.545949935913086,
|
|
"mean_logprob": -7.848649978637695,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -30.538150787353516,
|
|
"mean_logprob": -7.634537696838379,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -30.116004943847656,
|
|
"mean_logprob": -10.038668314615885,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.695979118347168,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02743094600009499
|
|
},
|
|
{
|
|
"id": "language_completion-030",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "The contract is valid provided that both parties",
|
|
"continuations": [
|
|
" had validity tomorrow.",
|
|
" sign it.",
|
|
" providing a party.",
|
|
" signs them."
|
|
],
|
|
"tags": [
|
|
"conditional"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -35.413047790527344,
|
|
"mean_logprob": -8.853261947631836,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.452535629272461,
|
|
"mean_logprob": -2.817511876424154,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -28.993125915527344,
|
|
"mean_logprob": -7.248281478881836,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -19.822309494018555,
|
|
"mean_logprob": -6.607436498006185,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.789924621582031,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028958492999663576
|
|
},
|
|
{
|
|
"id": "language_completion-031",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "No sooner had the bell rung than the students",
|
|
"continuations": [
|
|
" have ringing bells.",
|
|
" sooner the bell.",
|
|
" left the classroom.",
|
|
" had leave tomorrow."
|
|
],
|
|
"tags": [
|
|
"inversion"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -27.051218032836914,
|
|
"mean_logprob": -4.508536338806152,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -22.70492935180664,
|
|
"mean_logprob": -4.540985870361328,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.72513484954834,
|
|
"mean_logprob": -2.181283712387085,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -21.85160255432129,
|
|
"mean_logprob": -5.462900638580322,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.3272526264190674,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03914885999984108
|
|
},
|
|
{
|
|
"id": "language_completion-032",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "The evidence was incomplete. Consequently, the investigators",
|
|
"continuations": [
|
|
" completed evidence by definition.",
|
|
" investigated before consequence.",
|
|
" were the evidence.",
|
|
" withheld judgment."
|
|
],
|
|
"tags": [
|
|
"discourse"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -37.00300598144531,
|
|
"mean_logprob": -7.400601196289062,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -32.91265869140625,
|
|
"mean_logprob": -8.228164672851562,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -31.54528045654297,
|
|
"mean_logprob": -7.886320114135742,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.593650817871094,
|
|
"mean_logprob": -4.398412704467773,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.002188491821289,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.033153643998957705
|
|
},
|
|
{
|
|
"id": "language_completion-033",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "Walking along the shore, I noticed",
|
|
"continuations": [
|
|
" a trail of fresh footprints.",
|
|
" the shore walked me.",
|
|
" noticing had walked.",
|
|
" along was a footprint."
|
|
],
|
|
"tags": [
|
|
"modifier"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -24.315420150756836,
|
|
"mean_logprob": -3.4736314501081194,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -27.378658294677734,
|
|
"mean_logprob": -5.475731658935547,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -39.87729263305664,
|
|
"mean_logprob": -7.9754585266113285,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -41.18281936645508,
|
|
"mean_logprob": -6.863803227742513,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.0021002088274273,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03778556900033436
|
|
},
|
|
{
|
|
"id": "language_completion-034",
|
|
"category": "language_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.5,
|
|
"earned_points": 1.5,
|
|
"item_elo": 850,
|
|
"context": "The orchestra rehearsed the difficult passage repeatedly until it sounded",
|
|
"continuations": [
|
|
" passage by difficulty.",
|
|
" effortless.",
|
|
" rehearsing the sound.",
|
|
" repeatably orchestra."
|
|
],
|
|
"tags": [
|
|
"collocation"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -32.8645133972168,
|
|
"mean_logprob": -8.2161283493042,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.816623687744141,
|
|
"mean_logprob": -1.9388745625813801,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -31.051162719726562,
|
|
"mean_logprob": -4.435880388532366,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -39.944427490234375,
|
|
"mean_logprob": -6.6574045817057295,
|
|
"total_sequence_tokens": 22
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.4970058259509864,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04088619899994228
|
|
},
|
|
{
|
|
"id": "language_completion-035",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "What the witness omitted from her statement proved",
|
|
"continuations": [
|
|
" that omission witnessed.",
|
|
" proving had omitted.",
|
|
" more significant than what she included.",
|
|
" to include more omittedly."
|
|
],
|
|
"tags": [
|
|
"syntax"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -31.159378051757812,
|
|
"mean_logprob": -6.231875610351563,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -33.38984680175781,
|
|
"mean_logprob": -8.347461700439453,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -23.858367919921875,
|
|
"mean_logprob": -3.4083382742745534,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -43.83302307128906,
|
|
"mean_logprob": -7.305503845214844,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.8235373360770093,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05545987199911906
|
|
},
|
|
{
|
|
"id": "language_completion-036",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "Were the assumptions underlying the estimate to change, the conclusion would",
|
|
"continuations": [
|
|
" have assumptions beneath it.",
|
|
" changing was estimated.",
|
|
" underlie tomorrow.",
|
|
" need to be reconsidered."
|
|
],
|
|
"tags": [
|
|
"conditional"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -28.43362045288086,
|
|
"mean_logprob": -5.686724090576172,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -36.489532470703125,
|
|
"mean_logprob": -9.122383117675781,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -29.69447135925293,
|
|
"mean_logprob": -7.423617839813232,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -9.104352951049805,
|
|
"mean_logprob": -1.300621850149972,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.3861022404262,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029180166000514873
|
|
},
|
|
{
|
|
"id": "language_completion-037",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "The policy, while defensible in principle, is difficult to",
|
|
"continuations": [
|
|
" implement consistently.",
|
|
" principle the defense.",
|
|
" difficult implementation was.",
|
|
" consistency defending."
|
|
],
|
|
"tags": [
|
|
"concession"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.661409378051758,
|
|
"mean_logprob": -2.2204697926839194,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -35.245487213134766,
|
|
"mean_logprob": -8.811371803283691,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -46.78346633911133,
|
|
"mean_logprob": -11.695866584777832,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -36.44639587402344,
|
|
"mean_logprob": -9.11159896850586,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 6.5909020105997715,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027174740000191377
|
|
},
|
|
{
|
|
"id": "language_completion-038",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "Only after the archive was digitized did researchers",
|
|
"continuations": [
|
|
" digitized only before.",
|
|
" discover the missing correspondence.",
|
|
" archive the discovery did.",
|
|
" correspondingly research."
|
|
],
|
|
"tags": [
|
|
"inversion"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -35.61037826538086,
|
|
"mean_logprob": -7.1220756530761715,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -25.180889129638672,
|
|
"mean_logprob": -5.0361778259277346,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -38.086387634277344,
|
|
"mean_logprob": -7.617277526855469,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -29.758289337158203,
|
|
"mean_logprob": -7.439572334289551,
|
|
"total_sequence_tokens": 13
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.085897827148437,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.030827698999928543
|
|
},
|
|
{
|
|
"id": "language_completion-039",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "The editor objected less to the claim itself than to",
|
|
"continuations": [
|
|
" claims having objections.",
|
|
" itself being less.",
|
|
" the certainty with which it was stated.",
|
|
" stating an editor."
|
|
],
|
|
"tags": [
|
|
"comparison"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -23.438323974609375,
|
|
"mean_logprob": -3.9063873291015625,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -33.21238708496094,
|
|
"mean_logprob": -8.303096771240234,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 9,
|
|
"sum_logprob": -24.74079132080078,
|
|
"mean_logprob": -2.748976813422309,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -31.99425506591797,
|
|
"mean_logprob": -6.398851013183593,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.1574105156792536,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026268130000971723
|
|
},
|
|
{
|
|
"id": "language_completion-040",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "So subtle was the change in tone that few listeners",
|
|
"continuations": [
|
|
" made tone into change.",
|
|
" subtly were few.",
|
|
" had listened tomorrow.",
|
|
" noticed it."
|
|
],
|
|
"tags": [
|
|
"inversion"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -36.30894470214844,
|
|
"mean_logprob": -7.261788940429687,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -43.557823181152344,
|
|
"mean_logprob": -8.711564636230468,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -30.78145980834961,
|
|
"mean_logprob": -6.156291961669922,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -13.248579025268555,
|
|
"mean_logprob": -4.416193008422852,
|
|
"total_sequence_tokens": 13
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.7400989532470703,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026592488000460435
|
|
},
|
|
{
|
|
"id": "language_completion-041",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "The proposal is neither as costly as its critics allege nor as simple as its supporters",
|
|
"continuations": [
|
|
" suggest.",
|
|
" allegation was cost.",
|
|
" supporting simply.",
|
|
" critics and costly."
|
|
],
|
|
"tags": [
|
|
"parallelism"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.625497817993164,
|
|
"mean_logprob": -2.312748908996582,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -36.28232192993164,
|
|
"mean_logprob": -7.2564643859863285,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -27.830577850341797,
|
|
"mean_logprob": -9.276859283447266,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -37.068687438964844,
|
|
"mean_logprob": -9.267171859741211,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.9437154769897464,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0277293990002363
|
|
},
|
|
{
|
|
"id": "language_completion-042",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "Whatever reservations the panel may have had, they were not sufficient to",
|
|
"continuations": [
|
|
" reserve a panel seat.",
|
|
" block the appointment.",
|
|
" appoint sufficient reservations.",
|
|
" having been whatever."
|
|
],
|
|
"tags": [
|
|
"concession"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -20.758148193359375,
|
|
"mean_logprob": -4.151629638671875,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.865694046020508,
|
|
"mean_logprob": -3.716423511505127,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -24.439010620117188,
|
|
"mean_logprob": -4.8878021240234375,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -34.549068450927734,
|
|
"mean_logprob": -8.637267112731934,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4352061271667482,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05680329200004053
|
|
},
|
|
{
|
|
"id": "language_completion-043",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "Had the data been interpreted in isolation, the apparent trend might have",
|
|
"continuations": [
|
|
" isolated its data.",
|
|
" been interpreting apparent.",
|
|
" seemed more convincing.",
|
|
" trended the isolation."
|
|
],
|
|
"tags": [
|
|
"counterfactual"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -24.237958908081055,
|
|
"mean_logprob": -6.059489727020264,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -35.368247985839844,
|
|
"mean_logprob": -7.073649597167969,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -17.2215518951416,
|
|
"mean_logprob": -2.870258649190267,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -35.14228820800781,
|
|
"mean_logprob": -7.0284576416015625,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.1892310778299966,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.059677686000213725
|
|
},
|
|
{
|
|
"id": "language_completion-044",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "The mayor's apology did little to allay concerns that the inquiry had been",
|
|
"continuations": [
|
|
" mayoral and apologetic.",
|
|
" concerned into inquiry.",
|
|
" doing little apology.",
|
|
" deliberately delayed."
|
|
],
|
|
"tags": [
|
|
"collocation"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -28.935810089111328,
|
|
"mean_logprob": -4.1336871555873325,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -41.75049591064453,
|
|
"mean_logprob": -10.437623977661133,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -31.012474060058594,
|
|
"mean_logprob": -6.202494812011719,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.5513129234313965,
|
|
"mean_logprob": -1.6378282308578491,
|
|
"total_sequence_tokens": 22
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.4958589247294833,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04653508400042483
|
|
},
|
|
{
|
|
"id": "language_completion-045",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "Not until the final chapter does the significance of the opening scene become",
|
|
"continuations": [
|
|
" fully apparent.",
|
|
" final before opening.",
|
|
" chaptered significantly.",
|
|
" opening does final."
|
|
],
|
|
"tags": [
|
|
"inversion"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.648927211761475,
|
|
"mean_logprob": -2.5496424039204917,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -25.284252166748047,
|
|
"mean_logprob": -6.321063041687012,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -37.81638717651367,
|
|
"mean_logprob": -9.454096794128418,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -36.81341552734375,
|
|
"mean_logprob": -9.203353881835938,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.77142063776652,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027118156998767518
|
|
},
|
|
{
|
|
"id": "language_completion-046",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "The theory's appeal lies not in its novelty but in its ability to",
|
|
"continuations": [
|
|
" appeal theoretically.",
|
|
" unify previously separate observations.",
|
|
" become novel before theory.",
|
|
" observe its ability."
|
|
],
|
|
"tags": [
|
|
"semantics"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -18.210508346557617,
|
|
"mean_logprob": -3.6421016693115233,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -25.781158447265625,
|
|
"mean_logprob": -4.2968597412109375,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -40.01434326171875,
|
|
"mean_logprob": -8.00286865234375,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -31.227251052856445,
|
|
"mean_logprob": -7.806812763214111,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6547580718994142,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03853663200061419
|
|
},
|
|
{
|
|
"id": "language_completion-047",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "Much as the negotiators wanted an agreement, neither side was prepared to",
|
|
"continuations": [
|
|
" negotiate wanting.",
|
|
" agreement much as.",
|
|
" concede the central point.",
|
|
" prepare neither side."
|
|
],
|
|
"tags": [
|
|
"concession"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -23.181316375732422,
|
|
"mean_logprob": -4.636263275146485,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -35.157752990722656,
|
|
"mean_logprob": -8.789438247680664,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -22.38667106628418,
|
|
"mean_logprob": -3.1980958666120256,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -27.910926818847656,
|
|
"mean_logprob": -6.977731704711914,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.4381674085344591,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027152167000167537
|
|
},
|
|
{
|
|
"id": "language_completion-048",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "Whether the discrepancy reflects measurement error or a genuine anomaly remains",
|
|
"continuations": [
|
|
" measuring a reflection.",
|
|
" discrepant in error.",
|
|
" anomalously genuine.",
|
|
" uncertain."
|
|
],
|
|
"tags": [
|
|
"syntax"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -34.6393928527832,
|
|
"mean_logprob": -8.6598482131958,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -32.52997970581055,
|
|
"mean_logprob": -5.421663284301758,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -37.232521057128906,
|
|
"mean_logprob": -7.446504211425781,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -2.806727170944214,
|
|
"mean_logprob": -1.403363585472107,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.018299698829651,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02666539100027876
|
|
},
|
|
{
|
|
"id": "language_completion-049",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "The ruling established a precedent whose consequences would not become clear",
|
|
"continuations": [
|
|
" for several years.",
|
|
" precedent had ruled.",
|
|
" consequences establishing.",
|
|
" clearly whose."
|
|
],
|
|
"tags": [
|
|
"relative_clause"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.659148216247559,
|
|
"mean_logprob": -2.4147870540618896,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -32.767478942871094,
|
|
"mean_logprob": -6.553495788574219,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -32.46292495727539,
|
|
"mean_logprob": -10.820974985758463,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -32.78173828125,
|
|
"mean_logprob": -10.92724609375,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.1387087345123295,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026745708999442286
|
|
},
|
|
{
|
|
"id": "language_completion-050",
|
|
"category": "language_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.0,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.25,
|
|
"earned_points": 2.25,
|
|
"item_elo": 1000,
|
|
"context": "To attribute the recovery solely to the new policy would be to",
|
|
"continuations": [
|
|
" recover sole attribution.",
|
|
" ignore several contributing factors.",
|
|
" policy the recovery solely.",
|
|
" contribute an attribution."
|
|
],
|
|
"tags": [
|
|
"idiom"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -28.278736114501953,
|
|
"mean_logprob": -5.6557472229003904,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -23.966026306152344,
|
|
"mean_logprob": -4.793205261230469,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -29.296300888061523,
|
|
"mean_logprob": -5.859260177612304,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -25.130556106567383,
|
|
"mean_logprob": -5.026111221313476,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.23290596008300746,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027140629999848898
|
|
},
|
|
{
|
|
"id": "commonsense-001",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "A glass slipped from the table onto a tile floor. It most likely",
|
|
"continuations": [
|
|
" grew roots.",
|
|
" floated upward.",
|
|
" shattered.",
|
|
" became warmer than fire."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.35636329650879,
|
|
"mean_logprob": -5.785454432169597,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.766342163085938,
|
|
"mean_logprob": -4.441585540771484,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.128620147705078,
|
|
"mean_logprob": -2.0321550369262695,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -26.523273468017578,
|
|
"mean_logprob": -5.304654693603515,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.409430503845215,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.07991852200029825
|
|
},
|
|
{
|
|
"id": "commonsense-002",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "Jules wanted to mail a letter, so he put a stamp on the",
|
|
"continuations": [
|
|
" refrigerator.",
|
|
" envelope.",
|
|
" sidewalk.",
|
|
" spoon."
|
|
],
|
|
"tags": [
|
|
"everyday"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.811249732971191,
|
|
"mean_logprob": -2.702812433242798,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -0.9752203226089478,
|
|
"mean_logprob": -0.4876101613044739,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.075915336608887,
|
|
"mean_logprob": -2.7689788341522217,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.012920379638672,
|
|
"mean_logprob": -3.670973459879557,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.215202271938324,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0346436170002562
|
|
},
|
|
{
|
|
"id": "commonsense-003",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "The room was dark until Ana flipped the wall switch and the",
|
|
"continuations": [
|
|
" carpet vanished.",
|
|
" window froze.",
|
|
" lights came on.",
|
|
" ceiling became wet."
|
|
],
|
|
"tags": [
|
|
"causality"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -19.102373123168945,
|
|
"mean_logprob": -3.820474624633789,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.570735931396484,
|
|
"mean_logprob": -2.892683982849121,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.960273265838623,
|
|
"mean_logprob": -1.9900683164596558,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.106468200683594,
|
|
"mean_logprob": -4.276617050170898,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.9026156663894653,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028131573999417014
|
|
},
|
|
{
|
|
"id": "commonsense-004",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "Before slicing a loaf of bread, most people reach for",
|
|
"continuations": [
|
|
" a paintbrush.",
|
|
" a blanket.",
|
|
" a shoe.",
|
|
" a knife."
|
|
],
|
|
"tags": [
|
|
"tool_use"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.79000186920166,
|
|
"mean_logprob": -3.697500467300415,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.01634693145752,
|
|
"mean_logprob": -3.50408673286438,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.878457069396973,
|
|
"mean_logprob": -2.719614267349243,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -2.3763246536254883,
|
|
"mean_logprob": -0.5940811634063721,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.125533103942871,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.031615363999662804
|
|
},
|
|
{
|
|
"id": "commonsense-005",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "If a houseplant's soil is very dry, the plant probably needs",
|
|
"continuations": [
|
|
" water.",
|
|
" a password.",
|
|
" louder music.",
|
|
" a winter coat."
|
|
],
|
|
"tags": [
|
|
"everyday"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.1428794860839844,
|
|
"mean_logprob": -1.5714397430419922,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -22.261730194091797,
|
|
"mean_logprob": -7.420576731363933,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -34.00652313232422,
|
|
"mean_logprob": -6.801304626464844,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.291974067687988,
|
|
"mean_logprob": -3.822993516921997,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.251553773880005,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.06647472599979665
|
|
},
|
|
{
|
|
"id": "commonsense-006",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "Omar put leftovers in the refrigerator to keep them",
|
|
"continuations": [
|
|
" invisible.",
|
|
" cold.",
|
|
" expensive.",
|
|
" rectangular."
|
|
],
|
|
"tags": [
|
|
"everyday"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -14.542591094970703,
|
|
"mean_logprob": -7.271295547485352,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.701167106628418,
|
|
"mean_logprob": -1.850583553314209,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -16.922225952148438,
|
|
"mean_logprob": -8.461112976074219,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -17.653610229492188,
|
|
"mean_logprob": -8.826805114746094,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 5.420711994171143,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02647093199993833
|
|
},
|
|
{
|
|
"id": "commonsense-007",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "Someone standing outside in heavy rain without shelter will probably get",
|
|
"continuations": [
|
|
" taller.",
|
|
" younger.",
|
|
" wet.",
|
|
" magnetic."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.499820709228516,
|
|
"mean_logprob": -3.8332735697428384,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.593626976013184,
|
|
"mean_logprob": -6.796813488006592,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.4803338050842285,
|
|
"mean_logprob": -2.7401669025421143,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -16.12941551208496,
|
|
"mean_logprob": -8.06470775604248,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.0931066672007241,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028404538999893703
|
|
},
|
|
{
|
|
"id": "commonsense-008",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "A cyclist approaching a red traffic light should",
|
|
"continuations": [
|
|
" accelerate through every lane.",
|
|
" close both eyes.",
|
|
" remove the wheels.",
|
|
" stop."
|
|
],
|
|
"tags": [
|
|
"safety"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -21.939393997192383,
|
|
"mean_logprob": -3.1341991424560547,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -18.764877319335938,
|
|
"mean_logprob": -4.691219329833984,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -22.600740432739258,
|
|
"mean_logprob": -5.6501851081848145,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.850698471069336,
|
|
"mean_logprob": -2.425349235534668,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7088499069213867,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03384869600085949
|
|
},
|
|
{
|
|
"id": "commonsense-009",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "To find out whether someone has a fever, a nurse commonly uses",
|
|
"continuations": [
|
|
" a thermometer.",
|
|
" a ruler.",
|
|
" a compass.",
|
|
" a stapler."
|
|
],
|
|
"tags": [
|
|
"tool_use"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.429974555969238,
|
|
"mean_logprob": -1.8574936389923096,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.03149700164795,
|
|
"mean_logprob": -4.010499000549316,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.593362808227539,
|
|
"mean_logprob": -4.197787602742513,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -16.09111976623535,
|
|
"mean_logprob": -3.2182239532470702,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.3607303142547607,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.030042739999771584
|
|
},
|
|
{
|
|
"id": "commonsense-010",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 0.0,
|
|
"item_elo": 800,
|
|
"context": "When the phone battery reaches zero, the phone usually",
|
|
"continuations": [
|
|
" doubles in size.",
|
|
" turns off.",
|
|
" prints a book.",
|
|
" becomes waterproof."
|
|
],
|
|
"tags": [
|
|
"everyday"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -10.272951126098633,
|
|
"mean_logprob": -2.0545902252197266,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.171103000640869,
|
|
"mean_logprob": -2.0570343335469565,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -26.03506851196289,
|
|
"mean_logprob": -6.508767127990723,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.60114288330078,
|
|
"mean_logprob": -4.150285720825195,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.0024441083272299657,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03187223499844549
|
|
},
|
|
{
|
|
"id": "commonsense-011",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "A person who has not eaten all day is likely to feel",
|
|
"continuations": [
|
|
" polished.",
|
|
" transparent.",
|
|
" hungry.",
|
|
" metallic."
|
|
],
|
|
"tags": [
|
|
"human_needs"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -21.305147171020508,
|
|
"mean_logprob": -7.101715723673503,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -21.010116577148438,
|
|
"mean_logprob": -10.505058288574219,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.818660736083984,
|
|
"mean_logprob": -1.9395535786946614,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.83749008178711,
|
|
"mean_logprob": -5.279163360595703,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.339609781901042,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029702271000132896
|
|
},
|
|
{
|
|
"id": "commonsense-012",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "After washing wet clothes, a common next step is to",
|
|
"continuations": [
|
|
" bury them in concrete.",
|
|
" mail them overseas.",
|
|
" paint them black.",
|
|
" dry them."
|
|
],
|
|
"tags": [
|
|
"everyday"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -20.121253967285156,
|
|
"mean_logprob": -3.3535423278808594,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -22.138946533203125,
|
|
"mean_logprob": -4.427789306640625,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.949477195739746,
|
|
"mean_logprob": -3.4873692989349365,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.874208450317383,
|
|
"mean_logprob": -1.6247361501057942,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.7288061777750652,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0797503419998975
|
|
},
|
|
{
|
|
"id": "commonsense-013",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "If a candle flame is blown out, the room may become",
|
|
"continuations": [
|
|
" darker.",
|
|
" heavier than a car.",
|
|
" filled with snow.",
|
|
" twice as wide."
|
|
],
|
|
"tags": [
|
|
"causality"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.059157371520996,
|
|
"mean_logprob": -2.019719123840332,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -27.804567337036133,
|
|
"mean_logprob": -4.6340945561726885,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.497331619262695,
|
|
"mean_logprob": -3.374332904815674,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -22.085975646972656,
|
|
"mean_logprob": -5.521493911743164,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.3546137809753418,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.035416651000559796
|
|
},
|
|
{
|
|
"id": "commonsense-014",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "To unlock a typical front door from outside, a resident may use",
|
|
"continuations": [
|
|
" soap.",
|
|
" a key.",
|
|
" a pillow.",
|
|
" a forkful of rice."
|
|
],
|
|
"tags": [
|
|
"tool_use"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -15.688546180725098,
|
|
"mean_logprob": -7.844273090362549,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.20548152923584,
|
|
"mean_logprob": -2.4018271764119468,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.886775016784668,
|
|
"mean_logprob": -2.971693754196167,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -26.5023193359375,
|
|
"mean_logprob": -4.41705322265625,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5698665777842202,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03503433900004893
|
|
},
|
|
{
|
|
"id": "commonsense-015",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "Fresh snow on a sidewalk can make the surface",
|
|
"continuations": [
|
|
" spicy.",
|
|
" musical.",
|
|
" slippery.",
|
|
" edible."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.030292510986328,
|
|
"mean_logprob": -5.343430836995442,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -18.539226531982422,
|
|
"mean_logprob": -9.269613265991211,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -3.111222743988037,
|
|
"mean_logprob": -0.7778056859970093,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.036014556884766,
|
|
"mean_logprob": -3.6786715189615884,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.900865832964579,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03836592500010738
|
|
},
|
|
{
|
|
"id": "commonsense-016",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "If a smoke alarm sounds and smoke is visible, people should",
|
|
"continuations": [
|
|
" finish a long nap.",
|
|
" hide the alarm.",
|
|
" open a novel.",
|
|
" leave the building safely."
|
|
],
|
|
"tags": [
|
|
"safety"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -31.506465911865234,
|
|
"mean_logprob": -5.251077651977539,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.628789901733398,
|
|
"mean_logprob": -3.9071974754333496,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -35.51291275024414,
|
|
"mean_logprob": -8.878228187561035,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -13.135072708129883,
|
|
"mean_logprob": -2.6270145416259765,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.2801829338073731,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028125348999310518
|
|
},
|
|
{
|
|
"id": "commonsense-017",
|
|
"category": "commonsense",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "A ceramic mug filled with hot tea may be too hot to",
|
|
"continuations": [
|
|
" hold comfortably.",
|
|
" remember a name.",
|
|
" become ceramic.",
|
|
" count backward."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.978292465209961,
|
|
"mean_logprob": -3.4945731163024902,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -26.312808990478516,
|
|
"mean_logprob": -6.578202247619629,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -22.872270584106445,
|
|
"mean_logprob": -5.718067646026611,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -23.461456298828125,
|
|
"mean_logprob": -7.820485432942708,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.223494529724121,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03861710899946047
|
|
},
|
|
{
|
|
"id": "commonsense-018",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "Tara heard thunder only a few seconds after seeing lightning, suggesting the storm was",
|
|
"continuations": [
|
|
" beneath the ocean.",
|
|
" nearby.",
|
|
" made of glass.",
|
|
" over last year."
|
|
],
|
|
"tags": [
|
|
"causality"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.610671043395996,
|
|
"mean_logprob": -3.152667760848999,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.430370330810547,
|
|
"mean_logprob": -3.7151851654052734,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.351226806640625,
|
|
"mean_logprob": -3.5878067016601562,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -21.74671745300293,
|
|
"mean_logprob": -5.436679363250732,
|
|
"total_sequence_tokens": 22
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4351389408111572,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02832317799948214
|
|
},
|
|
{
|
|
"id": "commonsense-019",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "The freezer door was left open overnight. In the morning, the ice cream was probably",
|
|
"continuations": [
|
|
" sharper.",
|
|
" newly packaged.",
|
|
" soft or melted.",
|
|
" louder."
|
|
],
|
|
"tags": [
|
|
"causality"
|
|
],
|
|
"prediction": 3,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.146666526794434,
|
|
"mean_logprob": -3.7866666316986084,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -21.309471130371094,
|
|
"mean_logprob": -7.103157043457031,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -16.809839248657227,
|
|
"mean_logprob": -3.361967849731445,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.778133392333984,
|
|
"mean_logprob": -2.944533348083496,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.41743450164794904,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.035673541999130975
|
|
},
|
|
{
|
|
"id": "commonsense-020",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "Ravi arrived early for an interview but did not want to interrupt the meeting inside, so he",
|
|
"continuations": [
|
|
" shouted through the door.",
|
|
" canceled every meeting.",
|
|
" moved the building.",
|
|
" waited quietly."
|
|
],
|
|
"tags": [
|
|
"social"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -18.4698486328125,
|
|
"mean_logprob": -2.6385498046875,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -21.761749267578125,
|
|
"mean_logprob": -3.626958211263021,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.902360916137695,
|
|
"mean_logprob": -4.475590229034424,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -10.329994201660156,
|
|
"mean_logprob": -2.065998840332031,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5725509643554689,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.035349686000699876
|
|
},
|
|
{
|
|
"id": "commonsense-021",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "A recipe serves four people, but eight guests are expected. A sensible cook would",
|
|
"continuations": [
|
|
" double the ingredients.",
|
|
" halve every plate.",
|
|
" freeze the oven.",
|
|
" omit all food."
|
|
],
|
|
"tags": [
|
|
"planning"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.00696086883545,
|
|
"mean_logprob": -2.7517402172088623,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -22.688758850097656,
|
|
"mean_logprob": -4.537751770019531,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -24.844209671020508,
|
|
"mean_logprob": -6.211052417755127,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -22.738433837890625,
|
|
"mean_logprob": -4.547686767578125,
|
|
"total_sequence_tokens": 22
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.7860115528106686,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027442759999757982
|
|
},
|
|
{
|
|
"id": "commonsense-022",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "Mina's laptop became very hot and its fan ran constantly, so she first checked whether",
|
|
"continuations": [
|
|
" the keyboard knew her name.",
|
|
" its vents were blocked.",
|
|
" the screen was square.",
|
|
" noon had arrived."
|
|
],
|
|
"tags": [
|
|
"troubleshooting"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -23.518367767333984,
|
|
"mean_logprob": -3.9197279612223306,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -16.88454246520996,
|
|
"mean_logprob": -2.8140904108683267,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -22.184528350830078,
|
|
"mean_logprob": -4.4369056701660154,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -25.38568115234375,
|
|
"mean_logprob": -5.07713623046875,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.105637550354004,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027006951999283046
|
|
},
|
|
{
|
|
"id": "commonsense-023",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "A friend says, \"I just lost my job.\" The most considerate immediate response is to",
|
|
"continuations": [
|
|
" ask for a loan.",
|
|
" change the subject to sports.",
|
|
" express sympathy and listen.",
|
|
" congratulate them."
|
|
],
|
|
"tags": [
|
|
"social"
|
|
],
|
|
"prediction": 3,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -12.615581512451172,
|
|
"mean_logprob": -2.5231163024902346,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -27.88450813293457,
|
|
"mean_logprob": -4.647418022155762,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -18.08245277404785,
|
|
"mean_logprob": -3.013742129007975,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -11.871034622192383,
|
|
"mean_logprob": -2.3742069244384765,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.14890937805175808,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03359649000049103
|
|
},
|
|
{
|
|
"id": "commonsense-024",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "The label says a medicine may cause drowsiness. After taking it, a person should avoid",
|
|
"continuations": [
|
|
" drinking water.",
|
|
" reading the label.",
|
|
" sitting safely.",
|
|
" driving a car."
|
|
],
|
|
"tags": [
|
|
"safety"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.047860145568848,
|
|
"mean_logprob": -2.682620048522949,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.666956901550293,
|
|
"mean_logprob": -3.9167392253875732,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -23.298513412475586,
|
|
"mean_logprob": -7.766171137491862,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.242864608764648,
|
|
"mean_logprob": -3.560716152191162,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8780961036682129,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02827333200002613
|
|
},
|
|
{
|
|
"id": "commonsense-025",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "Water is dripping from the ceiling directly below an upstairs bathroom. The leak likely",
|
|
"continuations": [
|
|
" originates near that bathroom.",
|
|
" comes from sunlight.",
|
|
" proves the roof is dry.",
|
|
" makes water weightless."
|
|
],
|
|
"tags": [
|
|
"diagnosis"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -21.697128295898438,
|
|
"mean_logprob": -3.0995897565569197,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.447704315185547,
|
|
"mean_logprob": -4.111926078796387,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -27.732826232910156,
|
|
"mean_logprob": -4.622137705485026,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -34.71245574951172,
|
|
"mean_logprob": -6.942491149902343,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.012336322239467,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02621591200113471
|
|
},
|
|
{
|
|
"id": "commonsense-026",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "The car's fuel gauge is nearly empty and the next town is far away. The prudent action is to",
|
|
"continuations": [
|
|
" increase tire size.",
|
|
" refuel before leaving.",
|
|
" turn off the steering wheel.",
|
|
" ignore all signs."
|
|
],
|
|
"tags": [
|
|
"planning"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -23.950927734375,
|
|
"mean_logprob": -5.98773193359375,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -13.28018856048584,
|
|
"mean_logprob": -2.656037712097168,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.440576553344727,
|
|
"mean_logprob": -2.4915109361921037,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.294353485107422,
|
|
"mean_logprob": -3.8235883712768555,
|
|
"total_sequence_tokens": 28
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.16452677590506415,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03307416600000579
|
|
},
|
|
{
|
|
"id": "commonsense-027",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "A package marked \"fragile\" should generally be",
|
|
"continuations": [
|
|
" shaken to test it.",
|
|
" left in rain.",
|
|
" handled carefully.",
|
|
" used as a chair."
|
|
],
|
|
"tags": [
|
|
"everyday"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -24.553762435913086,
|
|
"mean_logprob": -4.092293739318848,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -21.365097045898438,
|
|
"mean_logprob": -5.341274261474609,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.0580472946167,
|
|
"mean_logprob": -3.3526824315389,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -23.24994468688965,
|
|
"mean_logprob": -4.6499889373779295,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7396113077799478,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027997896000670153
|
|
},
|
|
{
|
|
"id": "commonsense-028",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "Kai cannot hear the television, but everyone else can. Before replacing it, Kai should check",
|
|
"continuations": [
|
|
" whether the walls are blue.",
|
|
" how much the sofa weighs.",
|
|
" if tomorrow is Tuesday.",
|
|
" the volume and audio settings."
|
|
],
|
|
"tags": [
|
|
"troubleshooting"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -26.885547637939453,
|
|
"mean_logprob": -4.480924606323242,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -25.82662582397461,
|
|
"mean_logprob": -3.6895179748535156,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -20.93070411682129,
|
|
"mean_logprob": -4.186140823364258,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -16.532907485961914,
|
|
"mean_logprob": -2.7554845809936523,
|
|
"total_sequence_tokens": 26
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.9340333938598633,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03174066499923356
|
|
},
|
|
{
|
|
"id": "commonsense-029",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "A wet floor sign in a hallway warns pedestrians that they should",
|
|
"continuations": [
|
|
" walk carefully.",
|
|
" run faster.",
|
|
" turn off every light.",
|
|
" remove their shoes immediately."
|
|
],
|
|
"tags": [
|
|
"safety"
|
|
],
|
|
"prediction": 3,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.348014831542969,
|
|
"mean_logprob": -3.7826716105143228,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.512643814086914,
|
|
"mean_logprob": -3.837547938028971,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -27.827768325805664,
|
|
"mean_logprob": -5.565553665161133,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -9.475430488586426,
|
|
"mean_logprob": -1.8950860977172852,
|
|
"total_sequence_tokens": 22
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.8875855127970376,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02886738199958927
|
|
},
|
|
{
|
|
"id": "commonsense-030",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "Nico promised to return a borrowed book on Friday but needs it until Monday. The responsible choice is to",
|
|
"continuations": [
|
|
" hide the book.",
|
|
" ask the owner for an extension.",
|
|
" claim Friday was canceled.",
|
|
" lend it to a stranger."
|
|
],
|
|
"tags": [
|
|
"social"
|
|
],
|
|
"prediction": 3,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.196928024291992,
|
|
"mean_logprob": -3.299232006072998,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -20.57825469970703,
|
|
"mean_logprob": -2.9397506713867188,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -32.479103088378906,
|
|
"mean_logprob": -4.639871869768415,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -19.80704116821289,
|
|
"mean_logprob": -2.4758801460266113,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4638705253601074,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.033459694001066964
|
|
},
|
|
{
|
|
"id": "commonsense-031",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "A refrigerator is running, but food inside is warm. One useful first check is whether",
|
|
"continuations": [
|
|
" the food can read.",
|
|
" the kitchen is rectangular.",
|
|
" the door seals completely.",
|
|
" the shelves are named."
|
|
],
|
|
"tags": [
|
|
"troubleshooting"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -22.93718147277832,
|
|
"mean_logprob": -4.587436294555664,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -19.133255004882812,
|
|
"mean_logprob": -3.8266510009765624,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -16.214479446411133,
|
|
"mean_logprob": -2.702413241068522,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -28.418724060058594,
|
|
"mean_logprob": -4.736454010009766,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.1242377599080404,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02941785599978175
|
|
},
|
|
{
|
|
"id": "commonsense-032",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "A hiker sees dark clouds approaching and has several hours left on an exposed ridge. The safer choice is to",
|
|
"continuations": [
|
|
" climb the tallest tree.",
|
|
" hold a metal pole.",
|
|
" wait for lightning.",
|
|
" descend toward shelter."
|
|
],
|
|
"tags": [
|
|
"safety"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -18.073514938354492,
|
|
"mean_logprob": -3.0122524897257485,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -25.604442596435547,
|
|
"mean_logprob": -5.120888519287109,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -17.805259704589844,
|
|
"mean_logprob": -3.561051940917969,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -18.156234741210938,
|
|
"mean_logprob": -4.539058685302734,
|
|
"total_sequence_tokens": 28
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5487994511922203,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03707048300020688
|
|
},
|
|
{
|
|
"id": "commonsense-033",
|
|
"category": "commonsense",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "An egg dropped onto a thick cushion is less likely to break because the cushion",
|
|
"continuations": [
|
|
" reduces the sudden impact.",
|
|
" makes the egg colder.",
|
|
" removes gravity.",
|
|
" turns shell into rubber."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -19.05860710144043,
|
|
"mean_logprob": -3.811721420288086,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -20.780445098876953,
|
|
"mean_logprob": -3.463407516479492,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -20.995624542236328,
|
|
"mean_logprob": -6.998541514078776,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -35.17015838623047,
|
|
"mean_logprob": -7.034031677246094,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.34831390380859384,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.047453494000365026
|
|
},
|
|
{
|
|
"id": "commonsense-034",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "A metal jar lid is stuck. Running warm water over the lid can help because the metal may",
|
|
"continuations": [
|
|
" lose all mass.",
|
|
" expand slightly.",
|
|
" stop being metal.",
|
|
" become colder than ice."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -26.746746063232422,
|
|
"mean_logprob": -6.6866865158081055,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.445197105407715,
|
|
"mean_logprob": -2.4817323684692383,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -22.945758819580078,
|
|
"mean_logprob": -5.7364397048950195,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -18.385082244873047,
|
|
"mean_logprob": -3.064180374145508,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5824480056762695,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027411420000134967
|
|
},
|
|
{
|
|
"id": "commonsense-035",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "Lena receives an email demanding immediate payment through gift cards and threatening arrest. The message is most likely",
|
|
"continuations": [
|
|
" a routine tax receipt.",
|
|
" a weather alert.",
|
|
" a scam.",
|
|
" a library reminder."
|
|
],
|
|
"tags": [
|
|
"digital_safety"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -27.713577270507812,
|
|
"mean_logprob": -4.618929545084636,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.01344108581543,
|
|
"mean_logprob": -4.253360271453857,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.814584732055664,
|
|
"mean_logprob": -1.953646183013916,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -25.283443450927734,
|
|
"mean_logprob": -6.320860862731934,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.2997140884399414,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03003643500051112
|
|
},
|
|
{
|
|
"id": "commonsense-036",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "A sealed plastic bottle dents inward after hot air inside cools. The best explanation is that",
|
|
"continuations": [
|
|
" the bottle gained extra air.",
|
|
" cooling erased the plastic.",
|
|
" gravity reversed briefly.",
|
|
" pressure inside decreased."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -26.828231811523438,
|
|
"mean_logprob": -4.471371968587239,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -29.029586791992188,
|
|
"mean_logprob": -4.838264465332031,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -37.96666717529297,
|
|
"mean_logprob": -9.491666793823242,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -23.304729461669922,
|
|
"mean_logprob": -5.8261823654174805,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.36689249674479196,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027758326999901328
|
|
},
|
|
{
|
|
"id": "commonsense-037",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "A colleague repeatedly misses deadlines because requirements change without notice. The most constructive first step is to",
|
|
"continuations": [
|
|
" document changes and clarify priorities.",
|
|
" publicly insult the colleague.",
|
|
" silently redo all work forever.",
|
|
" delete the project."
|
|
],
|
|
"tags": [
|
|
"workplace"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -22.14117431640625,
|
|
"mean_logprob": -3.6901957194010415,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -24.30960464477539,
|
|
"mean_logprob": -3.4728006635393416,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 8,
|
|
"sum_logprob": -52.79871368408203,
|
|
"mean_logprob": -6.599839210510254,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.443693161010742,
|
|
"mean_logprob": -4.3609232902526855,
|
|
"total_sequence_tokens": 26
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.21739505586169994,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028819236000344972
|
|
},
|
|
{
|
|
"id": "commonsense-038",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "A carbon monoxide alarm sounds, but nobody can smell smoke. Occupants should still",
|
|
"continuations": [
|
|
" assume silence means safety.",
|
|
" leave and contact emergency services.",
|
|
" search with a flame.",
|
|
" unplug it and sleep."
|
|
],
|
|
"tags": [
|
|
"safety"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -34.20486831665039,
|
|
"mean_logprob": -6.840973663330078,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -21.1650333404541,
|
|
"mean_logprob": -3.52750555674235,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -27.941219329833984,
|
|
"mean_logprob": -4.656869888305664,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -27.197368621826172,
|
|
"mean_logprob": -3.885338374546596,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3578328178042458,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02605757099991024
|
|
},
|
|
{
|
|
"id": "commonsense-039",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "A wooden drawer sticks only during humid weather. The most plausible reason is that the wood",
|
|
"continuations": [
|
|
" becomes electrically charged.",
|
|
" forgets its shape.",
|
|
" swells after absorbing moisture.",
|
|
" turns into metal."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -19.181184768676758,
|
|
"mean_logprob": -3.196864128112793,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -23.117597579956055,
|
|
"mean_logprob": -4.623519515991211,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -30.014820098876953,
|
|
"mean_logprob": -4.2878314426967075,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -20.909446716308594,
|
|
"mean_logprob": -5.227361679077148,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.0909673145839145,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03314090099956957
|
|
},
|
|
{
|
|
"id": "commonsense-040",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "Sam hears a rumor about a coworker from someone who admits they were not present. Before repeating it, Sam should",
|
|
"continuations": [
|
|
" add dramatic details.",
|
|
" post it anonymously.",
|
|
" treat it as confirmed.",
|
|
" verify the information."
|
|
],
|
|
"tags": [
|
|
"social"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -29.395584106445312,
|
|
"mean_logprob": -7.348896026611328,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -21.053836822509766,
|
|
"mean_logprob": -3.5089728037516275,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -22.396867752075195,
|
|
"mean_logprob": -4.479373550415039,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.483184814453125,
|
|
"mean_logprob": -2.6207962036132812,
|
|
"total_sequence_tokens": 31
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8881766001383462,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04679363399918657
|
|
},
|
|
{
|
|
"id": "commonsense-041",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "Two identical wet towels are hung up, one spread flat and one crumpled. The spread towel usually dries first because it has",
|
|
"continuations": [
|
|
" more surface exposed to air.",
|
|
" less fabric.",
|
|
" no water initially.",
|
|
" a different mass forever."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -14.11756706237793,
|
|
"mean_logprob": -2.352927843729655,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.636102676391602,
|
|
"mean_logprob": -3.5453675587972007,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.9916934967041,
|
|
"mean_logprob": -4.497923374176025,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -29.345022201538086,
|
|
"mean_logprob": -5.869004440307617,
|
|
"total_sequence_tokens": 35
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.1924397150675459,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04707452600086981
|
|
},
|
|
{
|
|
"id": "commonsense-042",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "A meeting invite lists 3 p.m. in another time zone. To avoid arriving at the wrong time, a participant should",
|
|
"continuations": [
|
|
" ignore the time zone.",
|
|
" convert it to local time.",
|
|
" attend at every hour.",
|
|
" change the calendar date randomly."
|
|
],
|
|
"tags": [
|
|
"planning"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -17.2685604095459,
|
|
"mean_logprob": -3.4537120819091798,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -22.19762420654297,
|
|
"mean_logprob": -3.699604034423828,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -27.377838134765625,
|
|
"mean_logprob": -5.475567626953125,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -31.6478328704834,
|
|
"mean_logprob": -5.274638811747233,
|
|
"total_sequence_tokens": 33
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.24589195251464835,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04162225400068564
|
|
},
|
|
{
|
|
"id": "commonsense-043",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "A pan of oil catches fire on a stove. The cook should turn off the heat and",
|
|
"continuations": [
|
|
" pour water into the oil.",
|
|
" carry the burning pan outside.",
|
|
" cover the pan with a lid.",
|
|
" fan the flames."
|
|
],
|
|
"tags": [
|
|
"safety"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -14.20297622680664,
|
|
"mean_logprob": -2.3671627044677734,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -21.66763687133789,
|
|
"mean_logprob": -3.6112728118896484,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -10.082666397094727,
|
|
"mean_logprob": -1.4403809138706751,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -12.846022605895996,
|
|
"mean_logprob": -2.5692045211791994,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.9267817905970983,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028579490000993246
|
|
},
|
|
{
|
|
"id": "commonsense-044",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "After a software update, a printer stops appearing on one computer but works from others. The issue is most likely with",
|
|
"continuations": [
|
|
" all paper ever made.",
|
|
" the building's plumbing.",
|
|
" the printer's physical existence.",
|
|
" that computer's configuration."
|
|
],
|
|
"tags": [
|
|
"troubleshooting"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -44.12640380859375,
|
|
"mean_logprob": -8.82528076171875,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -24.53286361694336,
|
|
"mean_logprob": -3.06660795211792,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -23.706457138061523,
|
|
"mean_logprob": -3.386636734008789,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -15.543107986450195,
|
|
"mean_logprob": -2.590517997741699,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4760899543762207,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.033685976999549894
|
|
},
|
|
{
|
|
"id": "commonsense-045",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "A person feels dizzy after standing up very quickly. Sitting back down helps primarily by",
|
|
"continuations": [
|
|
" reducing the immediate risk of falling.",
|
|
" making the floor warmer.",
|
|
" stopping all circulation.",
|
|
" increasing their height."
|
|
],
|
|
"tags": [
|
|
"safety"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -16.934162139892578,
|
|
"mean_logprob": -2.419166019984654,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -19.554096221923828,
|
|
"mean_logprob": -3.910819244384766,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -20.506786346435547,
|
|
"mean_logprob": -5.126696586608887,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.24651527404785,
|
|
"mean_logprob": -4.311628818511963,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.4916532244001117,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03306646099917998
|
|
},
|
|
{
|
|
"id": "commonsense-046",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "A sealed bag of chips puffs up during a drive to high altitude because outside air pressure",
|
|
"continuations": [
|
|
" becomes much higher.",
|
|
" becomes lower.",
|
|
" changes the chips into gas.",
|
|
" disappears completely."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.810972213745117,
|
|
"mean_logprob": -3.4527430534362793,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.870441436767578,
|
|
"mean_logprob": -3.290147145589193,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -30.123023986816406,
|
|
"mean_logprob": -5.020503997802734,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.583572387695312,
|
|
"mean_logprob": -4.145893096923828,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.16259590784708644,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02561516199966718
|
|
},
|
|
{
|
|
"id": "commonsense-047",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "A manager asks for feedback but becomes defensive whenever criticism is offered. Employees will likely become",
|
|
"continuations": [
|
|
" more candid each time.",
|
|
" unable to remember work.",
|
|
" reluctant to speak honestly.",
|
|
" certain all criticism vanished."
|
|
],
|
|
"tags": [
|
|
"social"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -25.41448974609375,
|
|
"mean_logprob": -5.08289794921875,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -31.7397403717041,
|
|
"mean_logprob": -6.34794807434082,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 8,
|
|
"sum_logprob": -15.93793773651123,
|
|
"mean_logprob": -1.9922422170639038,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -42.678192138671875,
|
|
"mean_logprob": -7.1130320231119795,
|
|
"total_sequence_tokens": 26
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.090655732154846,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027545950999410707
|
|
},
|
|
{
|
|
"id": "commonsense-048",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "A parked car has one window fogged on the inside while the others are clear. This suggests moisture is concentrated",
|
|
"continuations": [
|
|
" inside the engine only.",
|
|
" on the road beneath.",
|
|
" in the sunlight.",
|
|
" near that window."
|
|
],
|
|
"tags": [
|
|
"diagnosis"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -18.106586456298828,
|
|
"mean_logprob": -3.6213172912597655,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -16.36924934387207,
|
|
"mean_logprob": -3.273849868774414,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.548701286315918,
|
|
"mean_logprob": -3.1371753215789795,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.029919624328613,
|
|
"mean_logprob": -3.0074799060821533,
|
|
"total_sequence_tokens": 28
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.12969541549682617,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.034464940001271316
|
|
},
|
|
{
|
|
"id": "commonsense-049",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "A child apologizes only after being caught and immediately repeats the behavior. The apology alone is weak evidence of",
|
|
"continuations": [
|
|
" genuine change.",
|
|
" having been caught.",
|
|
" the behavior occurring.",
|
|
" spoken language."
|
|
],
|
|
"tags": [
|
|
"social"
|
|
],
|
|
"prediction": 2,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.416661262512207,
|
|
"mean_logprob": -4.138887087504069,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.955171585083008,
|
|
"mean_logprob": -4.488792896270752,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.714911460876465,
|
|
"mean_logprob": -3.678727865219116,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -21.599285125732422,
|
|
"mean_logprob": -7.199761708577474,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4601592222849531,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03128207199915778
|
|
},
|
|
{
|
|
"id": "commonsense-050",
|
|
"category": "commonsense",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "A rechargeable battery lasts much less time in freezing weather but recovers indoors. The temporary loss is most plausibly due to",
|
|
"continuations": [
|
|
" permanent deletion of electricity.",
|
|
" slower chemical reactions in the cold.",
|
|
" the device becoming heavier.",
|
|
" indoor clocks running faster."
|
|
],
|
|
"tags": [
|
|
"physical"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -31.399362564086914,
|
|
"mean_logprob": -5.233227094014485,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -24.70549201965332,
|
|
"mean_logprob": -3.529356002807617,
|
|
"total_sequence_tokens": 37
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -23.229263305664062,
|
|
"mean_logprob": -3.8715438842773438,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -34.779884338378906,
|
|
"mean_logprob": -5.796647389729817,
|
|
"total_sequence_tokens": 36
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.34218788146972656,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04696313400017971
|
|
},
|
|
{
|
|
"id": "world_knowledge-001",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "The capital city of France is",
|
|
"continuations": [
|
|
" Vienna.",
|
|
" Madrid.",
|
|
" Paris.",
|
|
" Lisbon."
|
|
],
|
|
"tags": [
|
|
"geography"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.509716033935547,
|
|
"mean_logprob": -3.836572011311849,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.032500743865967,
|
|
"mean_logprob": -2.344166914621989,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.3560290336608887,
|
|
"mean_logprob": -1.6780145168304443,
|
|
"total_sequence_tokens": 8
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.288416862487793,
|
|
"mean_logprob": -2.3221042156219482,
|
|
"total_sequence_tokens": 10
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6440896987915039,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03533909900033905
|
|
},
|
|
{
|
|
"id": "world_knowledge-002",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 0.0,
|
|
"item_elo": 800,
|
|
"context": "The planet known for its prominent rings is",
|
|
"continuations": [
|
|
" Mercury.",
|
|
" Saturn.",
|
|
" Earth.",
|
|
" Mars."
|
|
],
|
|
"tags": [
|
|
"science"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.320735454559326,
|
|
"mean_logprob": -2.4402451515197754,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.424192428588867,
|
|
"mean_logprob": -4.212096214294434,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.013822555541992,
|
|
"mean_logprob": -3.006911277770996,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.402318954467773,
|
|
"mean_logprob": -4.201159477233887,
|
|
"total_sequence_tokens": 10
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5666661262512207,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027122282999698655
|
|
},
|
|
{
|
|
"id": "world_knowledge-003",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "Water freezes at zero degrees on the",
|
|
"continuations": [
|
|
" Kelvin scale.",
|
|
" Fahrenheit scale.",
|
|
" Celsius scale.",
|
|
" Beaufort scale."
|
|
],
|
|
"tags": [
|
|
"science"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -8.779051780700684,
|
|
"mean_logprob": -1.7558103561401368,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -10.190202713012695,
|
|
"mean_logprob": -1.4557432447160994,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -6.731854438781738,
|
|
"mean_logprob": -1.3463708877563476,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -15.025651931762695,
|
|
"mean_logprob": -2.504275321960449,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.10937235695975178,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02718069999900763
|
|
},
|
|
{
|
|
"id": "world_knowledge-004",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 0.0,
|
|
"item_elo": 800,
|
|
"context": "The author of the play Hamlet was",
|
|
"continuations": [
|
|
" Charles Dickens.",
|
|
" Jane Austen.",
|
|
" Mark Twain.",
|
|
" William Shakespeare."
|
|
],
|
|
"tags": [
|
|
"literature"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.206514358520508,
|
|
"mean_logprob": -2.801628589630127,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.495871543884277,
|
|
"mean_logprob": -3.1239678859710693,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.107165336608887,
|
|
"mean_logprob": -3.7767913341522217,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.643210411071777,
|
|
"mean_logprob": -3.5477368036905923,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3223392963409424,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026172885000050883
|
|
},
|
|
{
|
|
"id": "world_knowledge-005",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "The largest ocean on Earth is the",
|
|
"continuations": [
|
|
" Pacific Ocean.",
|
|
" Indian Ocean.",
|
|
" Arctic Ocean.",
|
|
" Atlantic Ocean."
|
|
],
|
|
"tags": [
|
|
"geography"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.973231792449951,
|
|
"mean_logprob": -1.6577439308166504,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.313286304473877,
|
|
"mean_logprob": -1.7710954348246257,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.450287342071533,
|
|
"mean_logprob": -1.8167624473571777,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.077324867248535,
|
|
"mean_logprob": -1.6924416224161785,
|
|
"total_sequence_tokens": 10
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.03469769159952807,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03793094100001326
|
|
},
|
|
{
|
|
"id": "world_knowledge-006",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "Plants primarily absorb water through their",
|
|
"continuations": [
|
|
" flowers.",
|
|
" roots.",
|
|
" seeds.",
|
|
" fruit."
|
|
],
|
|
"tags": [
|
|
"biology"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -12.8627290725708,
|
|
"mean_logprob": -6.4313645362854,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -1.4958585500717163,
|
|
"mean_logprob": -0.7479292750358582,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.364479064941406,
|
|
"mean_logprob": -5.682239532470703,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.464406967163086,
|
|
"mean_logprob": -6.732203483581543,
|
|
"total_sequence_tokens": 9
|
|
}
|
|
],
|
|
"best_vs_second_margin": 4.934310257434845,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.031080873000973952
|
|
},
|
|
{
|
|
"id": "world_knowledge-007",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "The chemical symbol for oxygen is",
|
|
"continuations": [
|
|
" Ox.",
|
|
" G.",
|
|
" O.",
|
|
" Og."
|
|
],
|
|
"tags": [
|
|
"chemistry"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.892958641052246,
|
|
"mean_logprob": -3.630986213684082,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.250329971313477,
|
|
"mean_logprob": -6.625164985656738,
|
|
"total_sequence_tokens": 8
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.206754684448242,
|
|
"mean_logprob": -2.103377342224121,
|
|
"total_sequence_tokens": 8
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.048662185668945,
|
|
"mean_logprob": -3.682887395222982,
|
|
"total_sequence_tokens": 9
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.527608871459961,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027357783999832463
|
|
},
|
|
{
|
|
"id": "world_knowledge-008",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "The Nile River flows through the continent of",
|
|
"continuations": [
|
|
" South America.",
|
|
" Australia.",
|
|
" Europe.",
|
|
" Africa."
|
|
],
|
|
"tags": [
|
|
"geography"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.3431458473205566,
|
|
"mean_logprob": -1.1143819491068523,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.791843414306641,
|
|
"mean_logprob": -3.3959217071533203,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.591670036315918,
|
|
"mean_logprob": -2.795835018157959,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -1.2561030387878418,
|
|
"mean_logprob": -0.6280515193939209,
|
|
"total_sequence_tokens": 10
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4863304297129314,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0381768369989004
|
|
},
|
|
{
|
|
"id": "world_knowledge-009",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "A triangle has",
|
|
"continuations": [
|
|
" three sides.",
|
|
" four sides.",
|
|
" five sides.",
|
|
" six sides."
|
|
],
|
|
"tags": [
|
|
"mathematics"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.732003211975098,
|
|
"mean_logprob": -1.9106677373250325,
|
|
"total_sequence_tokens": 6
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.198739051818848,
|
|
"mean_logprob": -3.0662463506062827,
|
|
"total_sequence_tokens": 6
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.230048179626465,
|
|
"mean_logprob": -3.4100160598754883,
|
|
"total_sequence_tokens": 6
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.581594467163086,
|
|
"mean_logprob": -3.1938648223876953,
|
|
"total_sequence_tokens": 6
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.1555786132812502,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.030920102000891347
|
|
},
|
|
{
|
|
"id": "world_knowledge-010",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "The organ that pumps blood through the human body is the",
|
|
"continuations": [
|
|
" liver.",
|
|
" heart.",
|
|
" lung.",
|
|
" stomach."
|
|
],
|
|
"tags": [
|
|
"biology"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.063324928283691,
|
|
"mean_logprob": -2.0316624641418457,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -1.312350869178772,
|
|
"mean_logprob": -0.656175434589386,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.319375038146973,
|
|
"mean_logprob": -4.659687519073486,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.700719833374023,
|
|
"mean_logprob": -3.8503599166870117,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.3754870295524597,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.024139918001310434
|
|
},
|
|
{
|
|
"id": "world_knowledge-011",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 0.0,
|
|
"item_elo": 800,
|
|
"context": "Japan is an island country in",
|
|
"continuations": [
|
|
" South America.",
|
|
" western Africa.",
|
|
" East Asia.",
|
|
" northern Europe."
|
|
],
|
|
"tags": [
|
|
"geography"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.345488548278809,
|
|
"mean_logprob": -1.7818295160929363,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.188445091247559,
|
|
"mean_logprob": -2.7294816970825195,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.010624885559082,
|
|
"mean_logprob": -2.3368749618530273,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.137768745422363,
|
|
"mean_logprob": -3.0459229151407876,
|
|
"total_sequence_tokens": 9
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5550454457600911,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05065048700089392
|
|
},
|
|
{
|
|
"id": "world_knowledge-012",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "The first month of the Gregorian calendar is",
|
|
"continuations": [
|
|
" March.",
|
|
" June.",
|
|
" December.",
|
|
" January."
|
|
],
|
|
"tags": [
|
|
"calendar"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.40371036529541,
|
|
"mean_logprob": -3.701855182647705,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.013602256774902,
|
|
"mean_logprob": -3.506801128387451,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.550854682922363,
|
|
"mean_logprob": -3.7754273414611816,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.948097229003906,
|
|
"mean_logprob": -2.974048614501953,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.532752513885498,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.071897378000358
|
|
},
|
|
{
|
|
"id": "world_knowledge-013",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "Bees commonly produce",
|
|
"continuations": [
|
|
" honey.",
|
|
" salt.",
|
|
" wool.",
|
|
" paper."
|
|
],
|
|
"tags": [
|
|
"biology"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.9243621826171875,
|
|
"mean_logprob": -2.9621810913085938,
|
|
"total_sequence_tokens": 6
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.732301712036133,
|
|
"mean_logprob": -5.866150856018066,
|
|
"total_sequence_tokens": 6
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.110575675964355,
|
|
"mean_logprob": -4.555287837982178,
|
|
"total_sequence_tokens": 6
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.831554412841797,
|
|
"mean_logprob": -5.915777206420898,
|
|
"total_sequence_tokens": 6
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.593106746673584,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03012902299997222
|
|
},
|
|
{
|
|
"id": "world_knowledge-014",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "The Great Pyramid of Giza is located in",
|
|
"continuations": [
|
|
" Peru.",
|
|
" Egypt.",
|
|
" India.",
|
|
" Greece."
|
|
],
|
|
"tags": [
|
|
"history"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -13.07826042175293,
|
|
"mean_logprob": -4.35942014058431,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.243206024169922,
|
|
"mean_logprob": -2.121603012084961,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.2263765335083,
|
|
"mean_logprob": -6.61318826675415,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -15.253130912780762,
|
|
"mean_logprob": -7.626565456390381,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.2378171284993487,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03506938899954548
|
|
},
|
|
{
|
|
"id": "world_knowledge-015",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 0.0,
|
|
"item_elo": 800,
|
|
"context": "The gas humans need to breathe for cellular respiration is",
|
|
"continuations": [
|
|
" helium.",
|
|
" neon.",
|
|
" oxygen.",
|
|
" argon."
|
|
],
|
|
"tags": [
|
|
"biology"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.004706382751465,
|
|
"mean_logprob": -3.001176595687866,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -22.805034637451172,
|
|
"mean_logprob": -7.601678212483724,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.715459823608398,
|
|
"mean_logprob": -3.357729911804199,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.974287986755371,
|
|
"mean_logprob": -3.991429328918457,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.356553316116333,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027213841000047978
|
|
},
|
|
{
|
|
"id": "world_knowledge-016",
|
|
"category": "world_knowledge",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.1,
|
|
"earned_points": 1.1,
|
|
"item_elo": 800,
|
|
"context": "A standard chessboard contains",
|
|
"continuations": [
|
|
" forty squares.",
|
|
" fifty squares.",
|
|
" one hundred squares.",
|
|
" sixty-four squares."
|
|
],
|
|
"tags": [
|
|
"culture"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.206071853637695,
|
|
"mean_logprob": -5.0686906178792315,
|
|
"total_sequence_tokens": 8
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.570915222167969,
|
|
"mean_logprob": -4.190305074055989,
|
|
"total_sequence_tokens": 8
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.034111022949219,
|
|
"mean_logprob": -3.7585277557373047,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.005380630493164,
|
|
"mean_logprob": -2.429340090070452,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.3291876656668529,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027590434001467656
|
|
},
|
|
{
|
|
"id": "world_knowledge-017",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The treaty that formally ended World War I between Germany and the Allied powers was the Treaty of",
|
|
"continuations": [
|
|
" Versailles.",
|
|
" Utrecht.",
|
|
" Tordesillas.",
|
|
" Ghent."
|
|
],
|
|
"tags": [
|
|
"history"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -0.7719295024871826,
|
|
"mean_logprob": -0.15438590049743653,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.48581600189209,
|
|
"mean_logprob": -2.8714540004730225,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -12.408638000488281,
|
|
"mean_logprob": -2.0681063334147134,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.774027824401855,
|
|
"mean_logprob": -2.924675941467285,
|
|
"total_sequence_tokens": 22
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.9137204329172768,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027041934999942896
|
|
},
|
|
{
|
|
"id": "world_knowledge-018",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The process by which plants convert light energy into chemical energy is",
|
|
"continuations": [
|
|
" fermentation.",
|
|
" photosynthesis.",
|
|
" respiration only.",
|
|
" erosion."
|
|
],
|
|
"tags": [
|
|
"biology"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.712061882019043,
|
|
"mean_logprob": -5.237353960673015,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.029754638671875,
|
|
"mean_logprob": -3.0074386596679688,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -25.57769203186035,
|
|
"mean_logprob": -6.394423007965088,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -20.068880081176758,
|
|
"mean_logprob": -10.034440040588379,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.229915301005046,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.031013443000119878
|
|
},
|
|
{
|
|
"id": "world_knowledge-019",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "The longest river entirely within Europe is the",
|
|
"continuations": [
|
|
" Danube.",
|
|
" Rhine.",
|
|
" Volga.",
|
|
" Seine."
|
|
],
|
|
"tags": [
|
|
"geography"
|
|
],
|
|
"prediction": 1,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.150540828704834,
|
|
"mean_logprob": -1.383513609568278,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.585611343383789,
|
|
"mean_logprob": -0.8618704477945963,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.604860782623291,
|
|
"mean_logprob": -1.868286927541097,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.248941898345947,
|
|
"mean_logprob": -2.0829806327819824,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5216431617736818,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03471019999960845
|
|
},
|
|
{
|
|
"id": "world_knowledge-020",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "In computing, RAM is primarily used for",
|
|
"continuations": [
|
|
" permanent archival storage.",
|
|
" printing documents.",
|
|
" cooling the processor.",
|
|
" temporary working memory."
|
|
],
|
|
"tags": [
|
|
"computing"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -20.72980308532715,
|
|
"mean_logprob": -4.1459606170654295,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.150440216064453,
|
|
"mean_logprob": -5.716813405354817,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.436466217041016,
|
|
"mean_logprob": -4.109116554260254,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.068862915039062,
|
|
"mean_logprob": -3.7672157287597656,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3419008255004883,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03145416299958015
|
|
},
|
|
{
|
|
"id": "world_knowledge-021",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "The scientist who formulated the three laws of motion was",
|
|
"continuations": [
|
|
" Isaac Newton.",
|
|
" Louis Pasteur.",
|
|
" Gregor Mendel.",
|
|
" Niels Bohr."
|
|
],
|
|
"tags": [
|
|
"science_history"
|
|
],
|
|
"prediction": 3,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.057504653930664,
|
|
"mean_logprob": -2.514376163482666,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -8.531913757324219,
|
|
"mean_logprob": -1.7063827514648438,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -10.613195419311523,
|
|
"mean_logprob": -1.7688659032185872,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -9.57681655883789,
|
|
"mean_logprob": -1.5961360931396484,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.11024665832519531,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.044894495000335155
|
|
},
|
|
{
|
|
"id": "world_knowledge-022",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The strait separating Spain from Morocco is the Strait of",
|
|
"continuations": [
|
|
" Malacca.",
|
|
" Gibraltar.",
|
|
" Hormuz.",
|
|
" Bering."
|
|
],
|
|
"tags": [
|
|
"geography"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.7044291496276855,
|
|
"mean_logprob": -1.4261072874069214,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -1.5994062423706055,
|
|
"mean_logprob": -0.3198812484741211,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -3.2809271812438965,
|
|
"mean_logprob": -0.8202317953109741,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.879630088806152,
|
|
"mean_logprob": -5.293210029602051,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.500350546836853,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03594739800064417
|
|
},
|
|
{
|
|
"id": "world_knowledge-023",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "A word with the opposite meaning of another word is called an",
|
|
"continuations": [
|
|
" acronym.",
|
|
" homophone.",
|
|
" antonym.",
|
|
" anagram."
|
|
],
|
|
"tags": [
|
|
"language"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.133785247802734,
|
|
"mean_logprob": -1.7834463119506836,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.040304660797119,
|
|
"mean_logprob": -1.7600761651992798,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -0.9590001106262207,
|
|
"mean_logprob": -0.31966670354207355,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.9414892196655273,
|
|
"mean_logprob": -0.9804964065551758,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6608297030131023,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029652658000486554
|
|
},
|
|
{
|
|
"id": "world_knowledge-024",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The part of a cell that contains most of its genetic material is the",
|
|
"continuations": [
|
|
" ribosome.",
|
|
" cell wall.",
|
|
" vacuole.",
|
|
" nucleus."
|
|
],
|
|
"tags": [
|
|
"biology"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.065311431884766,
|
|
"mean_logprob": -1.0163278579711914,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.278770446777344,
|
|
"mean_logprob": -3.7595901489257812,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.306262969970703,
|
|
"mean_logprob": -3.076565742492676,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -1.0450056791305542,
|
|
"mean_logprob": -0.5225028395652771,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4938250184059143,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02790486899903044
|
|
},
|
|
{
|
|
"id": "world_knowledge-025",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The Renaissance began as a major cultural movement in",
|
|
"continuations": [
|
|
" Italy.",
|
|
" Iceland.",
|
|
" Canada.",
|
|
" Mongolia."
|
|
],
|
|
"tags": [
|
|
"history"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.302576780319214,
|
|
"mean_logprob": -1.651288390159607,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.38290786743164,
|
|
"mean_logprob": -3.09572696685791,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.717884063720703,
|
|
"mean_logprob": -6.858942031860352,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -18.62806510925293,
|
|
"mean_logprob": -6.2093550364176435,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.4444385766983032,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028027561998897
|
|
},
|
|
{
|
|
"id": "world_knowledge-026",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The SI unit of electrical resistance is the",
|
|
"continuations": [
|
|
" watt.",
|
|
" ohm.",
|
|
" tesla.",
|
|
" joule."
|
|
],
|
|
"tags": [
|
|
"physics"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.065362930297852,
|
|
"mean_logprob": -4.688454310099284,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -1.9842658042907715,
|
|
"mean_logprob": -0.6614219347635905,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.778952598571777,
|
|
"mean_logprob": -3.6947381496429443,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.803125381469727,
|
|
"mean_logprob": -3.7007813453674316,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.033316214879354,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028208798999912688
|
|
},
|
|
{
|
|
"id": "world_knowledge-027",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The novel Pride and Prejudice was written by",
|
|
"continuations": [
|
|
" George Eliot.",
|
|
" Virginia Woolf.",
|
|
" Jane Austen.",
|
|
" Mary Shelley."
|
|
],
|
|
"tags": [
|
|
"literature"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.2252607345581055,
|
|
"mean_logprob": -1.5563151836395264,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -4.485273361206055,
|
|
"mean_logprob": -0.8970546722412109,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -2.3515846729278564,
|
|
"mean_logprob": -0.5878961682319641,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -8.91497802734375,
|
|
"mean_logprob": -1.78299560546875,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3091585040092468,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03339862100074242
|
|
},
|
|
{
|
|
"id": "world_knowledge-028",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The layer of Earth's atmosphere containing most weather is the",
|
|
"continuations": [
|
|
" mesosphere.",
|
|
" thermosphere.",
|
|
" exosphere.",
|
|
" troposphere."
|
|
],
|
|
"tags": [
|
|
"earth_science"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.590150833129883,
|
|
"mean_logprob": -1.8633836110432942,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.984744548797607,
|
|
"mean_logprob": -2.3282481829325357,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.345436096191406,
|
|
"mean_logprob": -2.4484786987304688,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -2.125917911529541,
|
|
"mean_logprob": -0.5314794778823853,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.331904133160909,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03057092500057479
|
|
},
|
|
{
|
|
"id": "world_knowledge-029",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The Magna Carta was sealed in",
|
|
"continuations": [
|
|
" England.",
|
|
" Brazil.",
|
|
" China.",
|
|
" Ethiopia."
|
|
],
|
|
"tags": [
|
|
"history"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.145002365112305,
|
|
"mean_logprob": -5.572501182556152,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -20.30511474609375,
|
|
"mean_logprob": -10.152557373046875,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -15.851038932800293,
|
|
"mean_logprob": -7.9255194664001465,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.8714599609375,
|
|
"mean_logprob": -5.623819986979167,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.05131880442301462,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04208292600014829
|
|
},
|
|
{
|
|
"id": "world_knowledge-030",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "In music, a group of three notes played together is commonly called a",
|
|
"continuations": [
|
|
" scale.",
|
|
" chord.",
|
|
" tempo.",
|
|
" clef."
|
|
],
|
|
"tags": [
|
|
"music"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.692070960998535,
|
|
"mean_logprob": -3.3460354804992676,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -2.7004072666168213,
|
|
"mean_logprob": -1.3502036333084106,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.314258575439453,
|
|
"mean_logprob": -4.104752858479817,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.103459358215332,
|
|
"mean_logprob": -4.034486452738444,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.995831847190857,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029503273000955232
|
|
},
|
|
{
|
|
"id": "world_knowledge-031",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "The blood vessels that generally carry blood away from the heart are",
|
|
"continuations": [
|
|
" capillaries only.",
|
|
" lymph nodes.",
|
|
" arteries.",
|
|
" alveoli."
|
|
],
|
|
"tags": [
|
|
"biology"
|
|
],
|
|
"prediction": 1,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -17.37689208984375,
|
|
"mean_logprob": -3.47537841796875,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.757102012634277,
|
|
"mean_logprob": -3.2523673375447593,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.300205230712891,
|
|
"mean_logprob": -3.6501026153564453,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.829465866088867,
|
|
"mean_logprob": -3.457366466522217,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.20499912897745753,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.055547699999806355
|
|
},
|
|
{
|
|
"id": "world_knowledge-032",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The decimal number ten is written in binary as",
|
|
"continuations": [
|
|
" 1001.",
|
|
" 1111.",
|
|
" 1100.",
|
|
" 1010."
|
|
],
|
|
"tags": [
|
|
"computing"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -7.5533013343811035,
|
|
"mean_logprob": -1.2588835557301838,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -14.360594749450684,
|
|
"mean_logprob": -2.3934324582417807,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -9.910626411437988,
|
|
"mean_logprob": -1.651771068572998,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -4.08213472366333,
|
|
"mean_logprob": -0.6803557872772217,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5785277684529622,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0371820000000298
|
|
},
|
|
{
|
|
"id": "world_knowledge-033",
|
|
"category": "world_knowledge",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.6500000000000001,
|
|
"earned_points": 1.6500000000000001,
|
|
"item_elo": 900,
|
|
"context": "The mountain range commonly used as part of the boundary between Europe and Asia is the",
|
|
"continuations": [
|
|
" Ural Mountains.",
|
|
" Andes.",
|
|
" Alps.",
|
|
" Atlas Mountains."
|
|
],
|
|
"tags": [
|
|
"geography"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.6374831199646,
|
|
"mean_logprob": -1.40937077999115,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.830323219299316,
|
|
"mean_logprob": -2.2767744064331055,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.304528713226318,
|
|
"mean_logprob": -1.4348429044087727,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.810175895690918,
|
|
"mean_logprob": -1.9525439739227295,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.02547212441762281,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026710719999755383
|
|
},
|
|
{
|
|
"id": "world_knowledge-034",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "The principle that no two electrons in an atom can share all four quantum numbers is the",
|
|
"continuations": [
|
|
" Doppler effect.",
|
|
" Pauli exclusion principle.",
|
|
" uncertainty of measurement.",
|
|
" Hubble law."
|
|
],
|
|
"tags": [
|
|
"physics"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -14.631590843200684,
|
|
"mean_logprob": -2.9263181686401367,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -6.522121429443359,
|
|
"mean_logprob": -1.3044242858886719,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -27.918291091918945,
|
|
"mean_logprob": -6.979572772979736,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -26.616729736328125,
|
|
"mean_logprob": -6.654182434082031,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.6218938827514648,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027726984000764787
|
|
},
|
|
{
|
|
"id": "world_knowledge-035",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "The 1648 agreements often associated with the modern state system are known as the Peace of",
|
|
"continuations": [
|
|
" Augsburg.",
|
|
" Paris.",
|
|
" Westphalia.",
|
|
" Nijmegen."
|
|
],
|
|
"tags": [
|
|
"history"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.747247695922852,
|
|
"mean_logprob": -1.186811923980713,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.567901849746704,
|
|
"mean_logprob": -1.783950924873352,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -1.6582977771759033,
|
|
"mean_logprob": -0.41457444429397583,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -6.829130172729492,
|
|
"mean_logprob": -1.3658260345458983,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7722374796867371,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02695665899955202
|
|
},
|
|
{
|
|
"id": "world_knowledge-036",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "In linguistics, the smallest unit that carries meaning is a",
|
|
"continuations": [
|
|
" phoneme in every case.",
|
|
" clause.",
|
|
" diphthong.",
|
|
" morpheme."
|
|
],
|
|
"tags": [
|
|
"linguistics"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -26.508514404296875,
|
|
"mean_logprob": -4.4180857340494795,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.442619800567627,
|
|
"mean_logprob": -3.2213099002838135,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -12.539463996887207,
|
|
"mean_logprob": -2.5078927993774416,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.539055347442627,
|
|
"mean_logprob": -1.1347638368606567,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.3731289625167848,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02788937600053032
|
|
},
|
|
{
|
|
"id": "world_knowledge-037",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "The enzyme that unwinds the DNA double helix during replication is",
|
|
"continuations": [
|
|
" helicase.",
|
|
" amylase.",
|
|
" lipase.",
|
|
" pepsin."
|
|
],
|
|
"tags": [
|
|
"biology"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.697218418121338,
|
|
"mean_logprob": -1.9243046045303345,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.000181198120117,
|
|
"mean_logprob": -2.7500452995300293,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.166723251342773,
|
|
"mean_logprob": -4.055574417114258,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.399234771728516,
|
|
"mean_logprob": -3.7997449239095054,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8257406949996948,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025740410999787855
|
|
},
|
|
{
|
|
"id": "world_knowledge-038",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "The economic concept describing the value of the best forgone alternative is",
|
|
"continuations": [
|
|
" sunk cost.",
|
|
" opportunity cost.",
|
|
" marginal revenue.",
|
|
" price elasticity."
|
|
],
|
|
"tags": [
|
|
"economics"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.04764461517334,
|
|
"mean_logprob": -3.761911153793335,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.921466827392578,
|
|
"mean_logprob": -2.640488942464193,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.147680282592773,
|
|
"mean_logprob": -5.382560094197591,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.714448928833008,
|
|
"mean_logprob": -3.928612232208252,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.121422211329142,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.07778051700006472
|
|
},
|
|
{
|
|
"id": "world_knowledge-039",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "The literary device in which an absent or imaginary person is directly addressed is",
|
|
"continuations": [
|
|
" metonymy.",
|
|
" synecdoche.",
|
|
" apostrophe.",
|
|
" enjambment."
|
|
],
|
|
"tags": [
|
|
"literature"
|
|
],
|
|
"prediction": 1,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.08511734008789,
|
|
"mean_logprob": -2.2712793350219727,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -10.178845405578613,
|
|
"mean_logprob": -2.0357690811157227,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -12.251620292663574,
|
|
"mean_logprob": -2.450324058532715,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -13.08072280883789,
|
|
"mean_logprob": -2.616144561767578,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.2355102539062499,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027328506999765523
|
|
},
|
|
{
|
|
"id": "world_knowledge-040",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "The ocean current that carries warm water across the North Atlantic is commonly called the",
|
|
"continuations": [
|
|
" Humboldt Current.",
|
|
" Benguela Current.",
|
|
" Oyashio Current.",
|
|
" Gulf Stream."
|
|
],
|
|
"tags": [
|
|
"earth_science"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -7.287515640258789,
|
|
"mean_logprob": -1.2145859400431316,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -9.331193923950195,
|
|
"mean_logprob": -1.5551989873250325,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -22.413164138793945,
|
|
"mean_logprob": -3.7355273564656577,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -0.9502784609794617,
|
|
"mean_logprob": -0.31675948699315387,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8978264530499778,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027548702000785852
|
|
},
|
|
{
|
|
"id": "world_knowledge-041",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "The alloy traditionally made primarily from copper and tin is",
|
|
"continuations": [
|
|
" bronze.",
|
|
" brass.",
|
|
" steel.",
|
|
" pewter."
|
|
],
|
|
"tags": [
|
|
"materials"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.973139762878418,
|
|
"mean_logprob": -3.657713254292806,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.976274490356445,
|
|
"mean_logprob": -2.992091496785482,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.525206565856934,
|
|
"mean_logprob": -5.262603282928467,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.067255020141602,
|
|
"mean_logprob": -3.7668137550354004,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6656217575073242,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02840788299909036
|
|
},
|
|
{
|
|
"id": "world_knowledge-042",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "The mathematical constant that is the base of natural logarithms is",
|
|
"continuations": [
|
|
" pi.",
|
|
" e.",
|
|
" phi.",
|
|
" tau."
|
|
],
|
|
"tags": [
|
|
"mathematics"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.910679340362549,
|
|
"mean_logprob": -3.4553396701812744,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.696976900100708,
|
|
"mean_logprob": -1.848488450050354,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -12.537050247192383,
|
|
"mean_logprob": -6.268525123596191,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -18.091705322265625,
|
|
"mean_logprob": -9.045852661132812,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.6068512201309204,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02433855900017079
|
|
},
|
|
{
|
|
"id": "world_knowledge-043",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "In databases, a key that uniquely identifies each row in a table is a",
|
|
"continuations": [
|
|
" foreign schema.",
|
|
" null index.",
|
|
" primary key.",
|
|
" stored procedure."
|
|
],
|
|
"tags": [
|
|
"computing"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -24.418195724487305,
|
|
"mean_logprob": -8.139398574829102,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.958831787109375,
|
|
"mean_logprob": -4.319610595703125,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.394084930419922,
|
|
"mean_logprob": -1.1313616434733074,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.721197128295898,
|
|
"mean_logprob": -3.2403990427652993,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.109037399291992,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02499480299957213
|
|
},
|
|
{
|
|
"id": "world_knowledge-044",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "The branch of philosophy concerned with the nature and limits of knowledge is",
|
|
"continuations": [
|
|
" aesthetics.",
|
|
" ethics.",
|
|
" ontology alone.",
|
|
" epistemology."
|
|
],
|
|
"tags": [
|
|
"philosophy"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.635092735290527,
|
|
"mean_logprob": -4.211697578430176,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -12.695484161376953,
|
|
"mean_logprob": -6.347742080688477,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -20.721094131469727,
|
|
"mean_logprob": -5.180273532867432,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -7.288669586181641,
|
|
"mean_logprob": -1.4577339172363282,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.753963661193848,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0514239969998016
|
|
},
|
|
{
|
|
"id": "world_knowledge-045",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "The element with atomic number 74 is",
|
|
"continuations": [
|
|
" tungsten.",
|
|
" titanium.",
|
|
" tantalum.",
|
|
" tellurium."
|
|
],
|
|
"tags": [
|
|
"chemistry"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -9.828885078430176,
|
|
"mean_logprob": -1.9657770156860352,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.466218948364258,
|
|
"mean_logprob": -3.4887396494547525,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -10.116809844970703,
|
|
"mean_logprob": -2.0233619689941404,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.358242988586426,
|
|
"mean_logprob": -2.8395607471466064,
|
|
"total_sequence_tokens": 13
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.0575849533081052,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027971211999101797
|
|
},
|
|
{
|
|
"id": "world_knowledge-046",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "The ancient writing system used for the Mycenaean Greek language is",
|
|
"continuations": [
|
|
" cuneiform.",
|
|
" Linear B.",
|
|
" ogham.",
|
|
" hieratic."
|
|
],
|
|
"tags": [
|
|
"history"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.766053199768066,
|
|
"mean_logprob": -2.1915132999420166,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.610755920410156,
|
|
"mean_logprob": -2.5369186401367188,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -20.13168716430664,
|
|
"mean_logprob": -5.03292179107666,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.041725158691406,
|
|
"mean_logprob": -3.0104312896728516,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.34540534019470215,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026163692000409355
|
|
},
|
|
{
|
|
"id": "world_knowledge-047",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "A star's spectral class in the sequence OBAFGKM is primarily related to its",
|
|
"continuations": [
|
|
" distance from Earth.",
|
|
" number of planets.",
|
|
" surface temperature.",
|
|
" age alone."
|
|
],
|
|
"tags": [
|
|
"astronomy"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.229168891906738,
|
|
"mean_logprob": -2.0572922229766846,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -18.440418243408203,
|
|
"mean_logprob": -4.610104560852051,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.304070949554443,
|
|
"mean_logprob": -1.7680236498514812,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.59379768371582,
|
|
"mean_logprob": -4.864599227905273,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.2892685731252034,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02666007100015122
|
|
},
|
|
{
|
|
"id": "world_knowledge-048",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "The grammatical case typically marking direct address is the",
|
|
"continuations": [
|
|
" accusative.",
|
|
" genitive.",
|
|
" instrumental.",
|
|
" vocative."
|
|
],
|
|
"tags": [
|
|
"linguistics"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.903148651123047,
|
|
"mean_logprob": -3.9757871627807617,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.181949615478516,
|
|
"mean_logprob": -5.060649871826172,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.94576072692871,
|
|
"mean_logprob": -5.98192024230957,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.720349311828613,
|
|
"mean_logprob": -5.240116437276204,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.0848627090454102,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03568337000069732
|
|
},
|
|
{
|
|
"id": "world_knowledge-049",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 2.475,
|
|
"item_elo": 1050,
|
|
"context": "The boundary around a black hole beyond which light cannot escape is the",
|
|
"continuations": [
|
|
" event horizon.",
|
|
" heliopause.",
|
|
" Roche limit.",
|
|
" photosphere."
|
|
],
|
|
"tags": [
|
|
"physics"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -1.2948393821716309,
|
|
"mean_logprob": -0.43161312739054364,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -7.611450672149658,
|
|
"mean_logprob": -1.5222901344299316,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.070302963256836,
|
|
"mean_logprob": -2.517575740814209,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.192702293395996,
|
|
"mean_logprob": -3.730900764465332,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.0906770070393879,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.030300678001367487
|
|
},
|
|
{
|
|
"id": "world_knowledge-050",
|
|
"category": "world_knowledge",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.1,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.475,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "In rhetoric, an argument that attacks the person rather than the claim is called",
|
|
"continuations": [
|
|
" reductio ad absurdum.",
|
|
" ad hominem.",
|
|
" modus ponens.",
|
|
" false equivalence only."
|
|
],
|
|
"tags": [
|
|
"rhetoric"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -8.812560081481934,
|
|
"mean_logprob": -1.1015700101852417,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -9.008734703063965,
|
|
"mean_logprob": -1.801746940612793,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -17.020626068115234,
|
|
"mean_logprob": -2.836771011352539,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -30.69535255432129,
|
|
"mean_logprob": -7.673838138580322,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7001769304275514,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.035687636000147904
|
|
},
|
|
{
|
|
"id": "context_tracking-001",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 1.2,
|
|
"item_elo": 850,
|
|
"context": "On the shelf, the red box contains tea and the blue box contains rice. The tea is in",
|
|
"continuations": [
|
|
" the red box.",
|
|
" the blue box.",
|
|
" both boxes.",
|
|
" neither box."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.783926486968994,
|
|
"mean_logprob": -1.6959816217422485,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.442936897277832,
|
|
"mean_logprob": -1.860734224319458,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.799026489257812,
|
|
"mean_logprob": -3.933008829752604,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -19.635343551635742,
|
|
"mean_logprob": -6.545114517211914,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.16475260257720947,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028946387999894796
|
|
},
|
|
{
|
|
"id": "context_tracking-002",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 1.2,
|
|
"item_elo": 850,
|
|
"context": "Ivo chose the violin, while Mara chose the flute. Mara's instrument is",
|
|
"continuations": [
|
|
" the violin.",
|
|
" unspecified.",
|
|
" the drum.",
|
|
" the flute."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.989151477813721,
|
|
"mean_logprob": -1.9972878694534302,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.466575622558594,
|
|
"mean_logprob": -6.733287811279297,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.59686279296875,
|
|
"mean_logprob": -3.8656209309895835,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.811376094818115,
|
|
"mean_logprob": -1.4528440237045288,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5444438457489014,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02690631699988444
|
|
},
|
|
{
|
|
"id": "context_tracking-003",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 1.2,
|
|
"item_elo": 850,
|
|
"context": "The morning bus stops at Oak Street. The evening bus stops at Pine Street. The evening stop is",
|
|
"continuations": [
|
|
" Oak Street.",
|
|
" Maple Street.",
|
|
" Pine Street.",
|
|
" Cedar Street."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.7684435844421387,
|
|
"mean_logprob": -1.256147861480713,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.26771354675293,
|
|
"mean_logprob": -2.0669283866882324,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.313518524169922,
|
|
"mean_logprob": -1.0783796310424805,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -9.800602912902832,
|
|
"mean_logprob": -1.9601205825805663,
|
|
"total_sequence_tokens": 26
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.17776823043823242,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029487604000678402
|
|
},
|
|
{
|
|
"id": "context_tracking-004",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 0.0,
|
|
"item_elo": 850,
|
|
"context": "Nia's badge number is 27, and Sol's badge number is 41. Badge 41 belongs to",
|
|
"continuations": [
|
|
" Nia.",
|
|
" both people.",
|
|
" nobody.",
|
|
" Sol."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.971212387084961,
|
|
"mean_logprob": -2.9904041290283203,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.68063735961914,
|
|
"mean_logprob": -5.56021245320638,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.08039379119873,
|
|
"mean_logprob": -4.693464597066243,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.419593334197998,
|
|
"mean_logprob": -3.709796667098999,
|
|
"total_sequence_tokens": 33
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7193925380706787,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0441296480003075
|
|
},
|
|
{
|
|
"id": "context_tracking-005",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 0.0,
|
|
"item_elo": 850,
|
|
"context": "The north garden grows mint. The south garden grows basil. Mint grows in",
|
|
"continuations": [
|
|
" the north garden.",
|
|
" the south garden.",
|
|
" both gardens.",
|
|
" the greenhouse."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.372170448303223,
|
|
"mean_logprob": -1.0930426120758057,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.286496639251709,
|
|
"mean_logprob": -1.0716241598129272,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.398435592651367,
|
|
"mean_logprob": -2.1328118642171225,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.518257141113281,
|
|
"mean_logprob": -1.8394190470377605,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.021418452262878418,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.07577670899991062
|
|
},
|
|
{
|
|
"id": "context_tracking-006",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 0.0,
|
|
"item_elo": 850,
|
|
"context": "Room A is painted yellow; Room B is painted green. The green room is",
|
|
"continuations": [
|
|
" Room A.",
|
|
" Room B.",
|
|
" both rooms.",
|
|
" Room C."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 3,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -19.31879425048828,
|
|
"mean_logprob": -6.439598083496094,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -20.013511657714844,
|
|
"mean_logprob": -6.671170552571614,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.426803588867188,
|
|
"mean_logprob": -5.1422678629557295,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.464043617248535,
|
|
"mean_logprob": -4.821347872416179,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3209199905395508,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.031925275001412956
|
|
},
|
|
{
|
|
"id": "context_tracking-007",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 1.2,
|
|
"item_elo": 850,
|
|
"context": "At noon, the cafe serves soup. At night, it serves noodles. The night menu features",
|
|
"continuations": [
|
|
" soup.",
|
|
" salad.",
|
|
" noodles.",
|
|
" bread only."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -15.040764808654785,
|
|
"mean_logprob": -7.520382404327393,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -18.535228729248047,
|
|
"mean_logprob": -6.178409576416016,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.340347290039062,
|
|
"mean_logprob": -3.5850868225097656,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -27.44425392150879,
|
|
"mean_logprob": -9.14808464050293,
|
|
"total_sequence_tokens": 26
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.59332275390625,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.08141225300096266
|
|
},
|
|
{
|
|
"id": "context_tracking-008",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 1.2,
|
|
"item_elo": 850,
|
|
"context": "Paz owns the bicycle, and Ren owns the scooter. The scooter belongs to",
|
|
"continuations": [
|
|
" Paz.",
|
|
" both Paz and Ren.",
|
|
" neither person.",
|
|
" Ren."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.017608642578125,
|
|
"mean_logprob": -2.6725362141927085,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -14.909689903259277,
|
|
"mean_logprob": -2.4849483172098794,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.485139846801758,
|
|
"mean_logprob": -5.828379948933919,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.9368906021118164,
|
|
"mean_logprob": -1.9684453010559082,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5165030161539712,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.035989186999358935
|
|
},
|
|
{
|
|
"id": "context_tracking-009",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 1.2,
|
|
"item_elo": 850,
|
|
"context": "The silver key opens the attic. The copper key opens the cellar. The attic key is",
|
|
"continuations": [
|
|
" silver.",
|
|
" copper.",
|
|
" wooden.",
|
|
" black."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.160383701324463,
|
|
"mean_logprob": -3.5801918506622314,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.434481620788574,
|
|
"mean_logprob": -4.717240810394287,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.718422889709473,
|
|
"mean_logprob": -4.859211444854736,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.321786880493164,
|
|
"mean_logprob": -4.160893440246582,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5807015895843506,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03191114499895775
|
|
},
|
|
{
|
|
"id": "context_tracking-010",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 1.2,
|
|
"item_elo": 850,
|
|
"context": "Toma ordered pears, while Eda ordered plums. Eda ordered",
|
|
"continuations": [
|
|
" pears.",
|
|
" plums.",
|
|
" peaches.",
|
|
" both fruits."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.681889057159424,
|
|
"mean_logprob": -1.2272963523864746,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.5526955127716064,
|
|
"mean_logprob": -1.1842318375905354,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.6570119857788086,
|
|
"mean_logprob": -1.2190039952596028,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.857694625854492,
|
|
"mean_logprob": -2.9525648752848306,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.03477215766906738,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029180496001572465
|
|
},
|
|
{
|
|
"id": "context_tracking-011",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 0.0,
|
|
"item_elo": 850,
|
|
"context": "The first train goes to Linz. The second goes to Graz. The second destination is",
|
|
"continuations": [
|
|
" Linz.",
|
|
" Vienna.",
|
|
" Graz.",
|
|
" Salzburg."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.7210230827331543,
|
|
"mean_logprob": -1.240341027577718,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.172287464141846,
|
|
"mean_logprob": -2.0574291547139487,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.512436866760254,
|
|
"mean_logprob": -1.8374789555867512,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.283622741699219,
|
|
"mean_logprob": -2.5709056854248047,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5971379280090332,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03351690399904328
|
|
},
|
|
{
|
|
"id": "context_tracking-012",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 0.0,
|
|
"item_elo": 850,
|
|
"context": "File K is dated March, and File M is dated June. The June file is",
|
|
"continuations": [
|
|
" File K.",
|
|
" both files.",
|
|
" undated.",
|
|
" File M."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 2,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.309968948364258,
|
|
"mean_logprob": -5.436656316121419,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -18.492210388183594,
|
|
"mean_logprob": -6.164070129394531,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.313602447509766,
|
|
"mean_logprob": -3.104534149169922,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -19.71459197998047,
|
|
"mean_logprob": -6.571530659993489,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.332122166951497,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.030803358000412118
|
|
},
|
|
{
|
|
"id": "context_tracking-013",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 1.2,
|
|
"item_elo": 850,
|
|
"context": "Luma wears a square pin. Oren wears a round pin. Luma's pin is",
|
|
"continuations": [
|
|
" square.",
|
|
" round.",
|
|
" triangular.",
|
|
" missing."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.6852593421936035,
|
|
"mean_logprob": -3.3426296710968018,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.037212371826172,
|
|
"mean_logprob": -4.018606185913086,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.829096794128418,
|
|
"mean_logprob": -4.414548397064209,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.15470027923584,
|
|
"mean_logprob": -3.57735013961792,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.23472046852111816,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026890174000072875
|
|
},
|
|
{
|
|
"id": "context_tracking-014",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 0.0,
|
|
"item_elo": 850,
|
|
"context": "The code for apple is 6. The code for orange is 9. Code 9 means",
|
|
"continuations": [
|
|
" apple.",
|
|
" orange.",
|
|
" pear.",
|
|
" fruit in general."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -12.063654899597168,
|
|
"mean_logprob": -6.031827449798584,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.191564559936523,
|
|
"mean_logprob": -6.595782279968262,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.624719619750977,
|
|
"mean_logprob": -5.541573206583659,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -25.945880889892578,
|
|
"mean_logprob": -6.4864702224731445,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.49025424321492483,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027514989000337664
|
|
},
|
|
{
|
|
"id": "context_tracking-015",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 1.2,
|
|
"item_elo": 850,
|
|
"context": "Kira arrived before Bo, and Bo arrived before Luis. The last to arrive was",
|
|
"continuations": [
|
|
" Kira.",
|
|
" Bo.",
|
|
" Luis.",
|
|
" nobody."
|
|
],
|
|
"tags": [
|
|
"order"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.463204383850098,
|
|
"mean_logprob": -2.1544014612833657,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.712535858154297,
|
|
"mean_logprob": -3.3562679290771484,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.134509563446045,
|
|
"mean_logprob": -1.0448365211486816,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.898235321044922,
|
|
"mean_logprob": -3.966078440348307,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.109564940134684,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027884559000085574
|
|
},
|
|
{
|
|
"id": "context_tracking-016",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 0.0,
|
|
"item_elo": 850,
|
|
"context": "The small parcel weighs two kilos. The large parcel weighs five. The heavier parcel is",
|
|
"continuations": [
|
|
" the small parcel.",
|
|
" both equally.",
|
|
" unknown.",
|
|
" the large parcel."
|
|
],
|
|
"tags": [
|
|
"comparison"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -12.038433074951172,
|
|
"mean_logprob": -2.4076866149902343,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -22.769210815429688,
|
|
"mean_logprob": -7.5897369384765625,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.203941345214844,
|
|
"mean_logprob": -5.601970672607422,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -13.794227600097656,
|
|
"mean_logprob": -2.7588455200195314,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.35115890502929714,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02827097000044887
|
|
},
|
|
{
|
|
"id": "context_tracking-017",
|
|
"category": "context_tracking",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.2,
|
|
"earned_points": 0.0,
|
|
"item_elo": 850,
|
|
"context": "Tuesday's password is comet. Wednesday's password is lantern. On Tuesday, the password is",
|
|
"continuations": [
|
|
" comet.",
|
|
" lantern.",
|
|
" Tuesday.",
|
|
" Wednesday."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.41854190826416,
|
|
"mean_logprob": -2.47284730275472,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.859280586242676,
|
|
"mean_logprob": -1.214820146560669,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.315832138061523,
|
|
"mean_logprob": -4.157916069030762,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.089353561401367,
|
|
"mean_logprob": -4.044676780700684,
|
|
"total_sequence_tokens": 26
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.2580271561940513,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03253696600040712
|
|
},
|
|
{
|
|
"id": "context_tracking-018",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "Three couriers carry different folders: Aya has violet, Ben has white, and Cy has black. The white folder is carried by",
|
|
"continuations": [
|
|
" Aya.",
|
|
" Ben.",
|
|
" Cy.",
|
|
" no courier."
|
|
],
|
|
"tags": [
|
|
"multi_binding"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.21363639831543,
|
|
"mean_logprob": -1.4045454661051433,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.199267864227295,
|
|
"mean_logprob": -3.0996339321136475,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.0412163734436035,
|
|
"mean_logprob": -3.5206081867218018,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.700834274291992,
|
|
"mean_logprob": -3.675208568572998,
|
|
"total_sequence_tokens": 32
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.6950884660085042,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.034491018999688094
|
|
},
|
|
{
|
|
"id": "context_tracking-019",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "The gallery opens after the cafe but before the theater. The theater opens after the library. Among the gallery, cafe, and theater, the earliest is",
|
|
"continuations": [
|
|
" the gallery.",
|
|
" the theater.",
|
|
" the cafe.",
|
|
" impossible to know."
|
|
],
|
|
"tags": [
|
|
"order"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.34194278717041,
|
|
"mean_logprob": -2.4473142623901367,
|
|
"total_sequence_tokens": 35
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.660017967224121,
|
|
"mean_logprob": -2.220005989074707,
|
|
"total_sequence_tokens": 35
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.666388034820557,
|
|
"mean_logprob": -1.4165970087051392,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -20.071624755859375,
|
|
"mean_logprob": -5.017906188964844,
|
|
"total_sequence_tokens": 36
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8034089803695679,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04306272899884789
|
|
},
|
|
{
|
|
"id": "context_tracking-020",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "A token moves from Box A to Box C, then from Box C to Box B. Its final location is",
|
|
"continuations": [
|
|
" Box A.",
|
|
" Box C.",
|
|
" every box.",
|
|
" Box B."
|
|
],
|
|
"tags": [
|
|
"state_update"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.557363986968994,
|
|
"mean_logprob": -0.852454662322998,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.336918354034424,
|
|
"mean_logprob": -1.7789727846781414,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -21.708280563354492,
|
|
"mean_logprob": -7.236093521118164,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.3088560104370117,
|
|
"mean_logprob": -0.7696186701456705,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.08283599217732751,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025915561000147136
|
|
},
|
|
{
|
|
"id": "context_tracking-021",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "Rin lends her map to Dev. Dev then gives the map back to Rin. At the end, the map is with",
|
|
"continuations": [
|
|
" Rin.",
|
|
" Dev.",
|
|
" both of them.",
|
|
" nobody."
|
|
],
|
|
"tags": [
|
|
"state_update"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.3433592319488525,
|
|
"mean_logprob": -0.7811197439829508,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.1370415687561035,
|
|
"mean_logprob": -2.0685207843780518,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.318612098693848,
|
|
"mean_logprob": -2.579653024673462,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.195390701293945,
|
|
"mean_logprob": -5.731796900431315,
|
|
"total_sequence_tokens": 30
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.287401040395101,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02816087800056266
|
|
},
|
|
{
|
|
"id": "context_tracking-022",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "The amber team scored below the teal team but above the gray team. The highest-scoring of these teams was",
|
|
"continuations": [
|
|
" amber.",
|
|
" teal.",
|
|
" gray.",
|
|
" tied."
|
|
],
|
|
"tags": [
|
|
"order"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.838001251220703,
|
|
"mean_logprob": -3.946000417073568,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -13.28606128692627,
|
|
"mean_logprob": -4.42868709564209,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -12.694733619689941,
|
|
"mean_logprob": -6.347366809844971,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.777946472167969,
|
|
"mean_logprob": -6.888973236083984,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.482686678568522,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029178336000768468
|
|
},
|
|
{
|
|
"id": "context_tracking-023",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "Ava's shift is Monday, Bram's is Thursday, and Cato's is Tuesday. The person working between Ava and Bram in the week is",
|
|
"continuations": [
|
|
" Ava.",
|
|
" Bram.",
|
|
" Cato.",
|
|
" nobody."
|
|
],
|
|
"tags": [
|
|
"calendar"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.949986457824707,
|
|
"mean_logprob": -2.316662152608236,
|
|
"total_sequence_tokens": 39
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.109921932220459,
|
|
"mean_logprob": -2.0366406440734863,
|
|
"total_sequence_tokens": 39
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.568081378936768,
|
|
"mean_logprob": -1.5226937929789226,
|
|
"total_sequence_tokens": 39
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.695931434631348,
|
|
"mean_logprob": -4.231977144877116,
|
|
"total_sequence_tokens": 39
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5139468510945637,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.046897251000700635
|
|
},
|
|
{
|
|
"id": "context_tracking-024",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "The note began under the lamp. Jo moved it to the desk, and Uma moved it from the desk into a drawer. The note ended",
|
|
"continuations": [
|
|
" under the lamp.",
|
|
" on the desk.",
|
|
" with Jo.",
|
|
" in the drawer."
|
|
],
|
|
"tags": [
|
|
"state_update"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.655835151672363,
|
|
"mean_logprob": -1.6639587879180908,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.382094383239746,
|
|
"mean_logprob": -2.5955235958099365,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.422002792358398,
|
|
"mean_logprob": -3.8073342641194663,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -7.003646373748779,
|
|
"mean_logprob": -1.400729274749756,
|
|
"total_sequence_tokens": 35
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.26322951316833487,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.047904680999636184
|
|
},
|
|
{
|
|
"id": "context_tracking-025",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "The labels are reassigned: north now means left, south means right, east means up, and west means down. Under this rule, north means",
|
|
"continuations": [
|
|
" left.",
|
|
" right.",
|
|
" up.",
|
|
" down."
|
|
],
|
|
"tags": [
|
|
"remapping"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.648298263549805,
|
|
"mean_logprob": -2.3241491317749023,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.928032398223877,
|
|
"mean_logprob": -2.9640161991119385,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.894533634185791,
|
|
"mean_logprob": -2.9472668170928955,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.785933494567871,
|
|
"mean_logprob": -2.8929667472839355,
|
|
"total_sequence_tokens": 32
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5688176155090332,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.034175955999671714
|
|
},
|
|
{
|
|
"id": "context_tracking-026",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "The bronze cup is taller than the glass cup. The glass cup is taller than the clay cup. The shortest is",
|
|
"continuations": [
|
|
" bronze.",
|
|
" clay.",
|
|
" glass.",
|
|
" not specified."
|
|
],
|
|
"tags": [
|
|
"transitive"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.209604263305664,
|
|
"mean_logprob": -3.0698680877685547,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.724428176879883,
|
|
"mean_logprob": -5.362214088439941,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.372103691101074,
|
|
"mean_logprob": -4.186051845550537,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.977124214172363,
|
|
"mean_logprob": -3.659041404724121,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5891733169555664,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02894302099957713
|
|
},
|
|
{
|
|
"id": "context_tracking-027",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "Nell assigns fox to 14, owl to 22, and hare to 31. The animal assigned 31 is",
|
|
"continuations": [
|
|
" fox.",
|
|
" owl.",
|
|
" hare.",
|
|
" none."
|
|
],
|
|
"tags": [
|
|
"binding"
|
|
],
|
|
"prediction": 1,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.879972457885742,
|
|
"mean_logprob": -5.439986228942871,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.412038803100586,
|
|
"mean_logprob": -3.470679601033529,
|
|
"total_sequence_tokens": 35
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.36612319946289,
|
|
"mean_logprob": -3.788707733154297,
|
|
"total_sequence_tokens": 35
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -14.26858139038086,
|
|
"mean_logprob": -7.13429069519043,
|
|
"total_sequence_tokens": 34
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3180281321207681,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03798832400025276
|
|
},
|
|
{
|
|
"id": "context_tracking-028",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "A meeting moves from Room 3 to Room 8, then is moved back to its original room. It will occur in",
|
|
"continuations": [
|
|
" Room 8.",
|
|
" both rooms.",
|
|
" no room.",
|
|
" Room 3."
|
|
],
|
|
"tags": [
|
|
"state_update"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.633544921875,
|
|
"mean_logprob": -1.15838623046875,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.276191711425781,
|
|
"mean_logprob": -3.4253972371419272,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.473928451538086,
|
|
"mean_logprob": -5.491309483846028,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.828451156616211,
|
|
"mean_logprob": -1.7071127891540527,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5487265586853027,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03693265699985204
|
|
},
|
|
{
|
|
"id": "context_tracking-029",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "In a made-up calendar, Dax follows Rul, and Vem follows Dax. The day immediately after Rul is",
|
|
"continuations": [
|
|
" Dax.",
|
|
" Vem.",
|
|
" Rul.",
|
|
" unknown."
|
|
],
|
|
"tags": [
|
|
"local_rules"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.311236381530762,
|
|
"mean_logprob": -2.4370787938435874,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.153491973876953,
|
|
"mean_logprob": -2.051163991292318,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.810539245605469,
|
|
"mean_logprob": -3.2701797485351562,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.88333797454834,
|
|
"mean_logprob": -3.94166898727417,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.38591480255126953,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03235356599907391
|
|
},
|
|
{
|
|
"id": "context_tracking-030",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "Faye is older than Gus but younger than Hana. The oldest of the three is",
|
|
"continuations": [
|
|
" Faye.",
|
|
" Hana.",
|
|
" Gus.",
|
|
" tied."
|
|
],
|
|
"tags": [
|
|
"transitive"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.905505180358887,
|
|
"mean_logprob": -1.7263762950897217,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.8858604431152344,
|
|
"mean_logprob": -1.2952868143717449,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.9375972747802734,
|
|
"mean_logprob": -1.3125324249267578,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -12.933774948120117,
|
|
"mean_logprob": -6.466887474060059,
|
|
"total_sequence_tokens": 22
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.017245610555012947,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04053015899989987
|
|
},
|
|
{
|
|
"id": "context_tracking-031",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "The red route reaches Delta before Echo. The blue route reaches Echo before Delta. On the blue route, the first of those stops is",
|
|
"continuations": [
|
|
" Delta.",
|
|
" neither stop.",
|
|
" Echo.",
|
|
" both together."
|
|
],
|
|
"tags": [
|
|
"conditional_binding"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.452212810516357,
|
|
"mean_logprob": -3.2261064052581787,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -18.530963897705078,
|
|
"mean_logprob": -6.176987965901692,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.69672155380249,
|
|
"mean_logprob": -1.5655738512674968,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -21.00954818725586,
|
|
"mean_logprob": -7.003182729085286,
|
|
"total_sequence_tokens": 32
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.660532553990682,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03073819699966407
|
|
},
|
|
{
|
|
"id": "context_tracking-032",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "Iris gives the ring to Juno, Juno gives it to Kato, and Kato keeps it. The ring is finally with",
|
|
"continuations": [
|
|
" Iris.",
|
|
" Juno.",
|
|
" everyone.",
|
|
" Kato."
|
|
],
|
|
"tags": [
|
|
"state_update"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.189102172851562,
|
|
"mean_logprob": -3.729700724283854,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.019768714904785,
|
|
"mean_logprob": -2.673256238301595,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.497590065002441,
|
|
"mean_logprob": -5.248795032501221,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.808121681213379,
|
|
"mean_logprob": -2.6027072270711265,
|
|
"total_sequence_tokens": 31
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.07054901123046875,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026309938000849797
|
|
},
|
|
{
|
|
"id": "context_tracking-033",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 1.7999999999999998,
|
|
"item_elo": 950,
|
|
"context": "For this report, green means urgent and red means routine. A green item is therefore",
|
|
"continuations": [
|
|
" urgent.",
|
|
" routine.",
|
|
" canceled.",
|
|
" unclassified."
|
|
],
|
|
"tags": [
|
|
"remapping"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.035861015319824,
|
|
"mean_logprob": -2.6786203384399414,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.059852600097656,
|
|
"mean_logprob": -5.529926300048828,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.928162574768066,
|
|
"mean_logprob": -3.9820406436920166,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.694698333740234,
|
|
"mean_logprob": -4.423674583435059,
|
|
"total_sequence_tokens": 22
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.3034203052520752,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026921026001218706
|
|
},
|
|
{
|
|
"id": "context_tracking-034",
|
|
"category": "context_tracking",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.7999999999999998,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "The oak is east of the pond. The bench is east of the oak. Relative to the pond, the bench is",
|
|
"continuations": [
|
|
" west.",
|
|
" east.",
|
|
" north.",
|
|
" at the same place."
|
|
],
|
|
"tags": [
|
|
"spatial"
|
|
],
|
|
"prediction": 3,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.630821228027344,
|
|
"mean_logprob": -4.315410614013672,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.045670509338379,
|
|
"mean_logprob": -4.0228352546691895,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.104247093200684,
|
|
"mean_logprob": -3.552123546600342,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -17.643739700317383,
|
|
"mean_logprob": -3.5287479400634765,
|
|
"total_sequence_tokens": 31
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.023375606536865323,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028784793001250364
|
|
},
|
|
{
|
|
"id": "context_tracking-035",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1100,
|
|
"context": "Four files are ordered J, K, L, M. K is moved after M, and J is then moved between L and M. The resulting order is",
|
|
"continuations": [
|
|
" J, L, M, K.",
|
|
" L, M, J, K.",
|
|
" L, J, M, K.",
|
|
" M, L, J, K."
|
|
],
|
|
"tags": [
|
|
"state_update"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -5.9900970458984375,
|
|
"mean_logprob": -0.7487621307373047,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -8.535561561584473,
|
|
"mean_logprob": -1.066945195198059,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 8,
|
|
"sum_logprob": -8.659443855285645,
|
|
"mean_logprob": -1.0824304819107056,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -11.459959030151367,
|
|
"mean_logprob": -1.432494878768921,
|
|
"total_sequence_tokens": 40
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3181830644607544,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.046220570999139454
|
|
},
|
|
{
|
|
"id": "context_tracking-036",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1100,
|
|
"context": "In a codebook, tulip means north, north means copper, and copper means seven. Following the chain from tulip leads to",
|
|
"continuations": [
|
|
" north only.",
|
|
" tulip.",
|
|
" copper only.",
|
|
" seven."
|
|
],
|
|
"tags": [
|
|
"multi_hop"
|
|
],
|
|
"prediction": 1,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.177152633666992,
|
|
"mean_logprob": -5.725717544555664,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.78298568725586,
|
|
"mean_logprob": -2.945746421813965,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.028417587280273,
|
|
"mean_logprob": -5.009472529093425,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.33794116973877,
|
|
"mean_logprob": -5.668970584869385,
|
|
"total_sequence_tokens": 31
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.06372610727946,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03617483699963486
|
|
},
|
|
{
|
|
"id": "context_tracking-037",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1100,
|
|
"context": "A cube starts in Bin R. It moves to S unless the flag is blue. The flag is blue, so it moves to T instead. Its final bin is",
|
|
"continuations": [
|
|
" Bin T.",
|
|
" Bin S.",
|
|
" Bin R.",
|
|
" unknown."
|
|
],
|
|
"tags": [
|
|
"conditional_update"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.813809394836426,
|
|
"mean_logprob": -3.604603131612142,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.571236610412598,
|
|
"mean_logprob": -2.523745536804199,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.749302864074707,
|
|
"mean_logprob": -3.249767621358236,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.8441290855407715,
|
|
"mean_logprob": -3.4220645427703857,
|
|
"total_sequence_tokens": 35
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7260220845540366,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03776685400043789
|
|
},
|
|
{
|
|
"id": "context_tracking-038",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1100,
|
|
"context": "Mira stands left of Noor. Olin stands right of Noor but left of Peta. From left to right, the second person is",
|
|
"continuations": [
|
|
" Mira.",
|
|
" Noor.",
|
|
" Olin.",
|
|
" Peta."
|
|
],
|
|
"tags": [
|
|
"spatial"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.994318962097168,
|
|
"mean_logprob": -1.6647729873657227,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.311159610748291,
|
|
"mean_logprob": -1.4370532035827637,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.769989252090454,
|
|
"mean_logprob": -0.923329750696818,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.6630380153656006,
|
|
"mean_logprob": -1.2210126717885335,
|
|
"total_sequence_tokens": 33
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.29768292109171546,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.034288531000129296
|
|
},
|
|
{
|
|
"id": "context_tracking-039",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 2.6999999999999997,
|
|
"item_elo": 1100,
|
|
"context": "The archive uses rotating labels. On day one A means red; on day two A means blue; on day three A means gold. Today is day three, so A means",
|
|
"continuations": [
|
|
" red.",
|
|
" blue.",
|
|
" gold.",
|
|
" every color."
|
|
],
|
|
"tags": [
|
|
"conditional_binding"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.996253967285156,
|
|
"mean_logprob": -2.498126983642578,
|
|
"total_sequence_tokens": 37
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.155964374542236,
|
|
"mean_logprob": -2.077982187271118,
|
|
"total_sequence_tokens": 37
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -0.3302699327468872,
|
|
"mean_logprob": -0.1651349663734436,
|
|
"total_sequence_tokens": 37
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.863689422607422,
|
|
"mean_logprob": -3.287896474202474,
|
|
"total_sequence_tokens": 38
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.9128472208976746,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04565987499881885
|
|
},
|
|
{
|
|
"id": "context_tracking-040",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1100,
|
|
"context": "A coin begins with Lio, passes to Mina, returns to Lio, then passes to Niko. Niko gives it to the person who held it immediately before him. The coin ends with",
|
|
"continuations": [
|
|
" Mina.",
|
|
" Niko.",
|
|
" nobody.",
|
|
" Lio."
|
|
],
|
|
"tags": [
|
|
"state_update"
|
|
],
|
|
"prediction": 1,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -1.7303788661956787,
|
|
"mean_logprob": -0.5767929553985596,
|
|
"total_sequence_tokens": 46
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -1.7619781494140625,
|
|
"mean_logprob": -0.4404945373535156,
|
|
"total_sequence_tokens": 47
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -12.21472454071045,
|
|
"mean_logprob": -4.071574846903483,
|
|
"total_sequence_tokens": 46
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.2031490802764893,
|
|
"mean_logprob": -0.7343830267588297,
|
|
"total_sequence_tokens": 46
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.13629841804504395,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04590733899931365
|
|
},
|
|
{
|
|
"id": "context_tracking-041",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 2.6999999999999997,
|
|
"item_elo": 1100,
|
|
"context": "In a race, Uma finished before Vic. Wes finished after Xena but before Uma. The earliest among Uma, Vic, Wes, and Xena was",
|
|
"continuations": [
|
|
" Xena.",
|
|
" Wes.",
|
|
" Uma.",
|
|
" Vic."
|
|
],
|
|
"tags": [
|
|
"transitive"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.8171894550323486,
|
|
"mean_logprob": -0.9390631516774496,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.756889820098877,
|
|
"mean_logprob": -1.2522966066996257,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.0636844635009766,
|
|
"mean_logprob": -1.0212281545003254,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.9644932746887207,
|
|
"mean_logprob": -0.9881644248962402,
|
|
"total_sequence_tokens": 41
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.04910127321879065,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04572442499920726
|
|
},
|
|
{
|
|
"id": "context_tracking-042",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 2.6999999999999997,
|
|
"item_elo": 1100,
|
|
"context": "A ledger maps 4 to pine, pine to west, and west to 19. Looking up 4 repeatedly until reaching a number returns",
|
|
"continuations": [
|
|
" west.",
|
|
" 19.",
|
|
" pine.",
|
|
" 4."
|
|
],
|
|
"tags": [
|
|
"multi_hop"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.747142791748047,
|
|
"mean_logprob": -5.873571395874023,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.2428412437438965,
|
|
"mean_logprob": -1.0607103109359741,
|
|
"total_sequence_tokens": 35
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.676970481872559,
|
|
"mean_logprob": -4.838485240936279,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.3904032707214355,
|
|
"mean_logprob": -2.130134423573812,
|
|
"total_sequence_tokens": 34
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.0694241126378379,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03764528200008499
|
|
},
|
|
{
|
|
"id": "context_tracking-043",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1100,
|
|
"context": "The black card outranks the white card. The striped card ranks below white but above gray. The second-highest card is",
|
|
"continuations": [
|
|
" black.",
|
|
" striped.",
|
|
" white.",
|
|
" gray."
|
|
],
|
|
"tags": [
|
|
"order"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.5012102127075195,
|
|
"mean_logprob": -1.7506051063537598,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.898595809936523,
|
|
"mean_logprob": -2.9661986033121743,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.047337532043457,
|
|
"mean_logprob": -2.0236687660217285,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.855133056640625,
|
|
"mean_logprob": -2.9275665283203125,
|
|
"total_sequence_tokens": 30
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.27306365966796875,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03667803600001207
|
|
},
|
|
{
|
|
"id": "context_tracking-044",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1100,
|
|
"context": "A marker is in Drawer A. Rule one moves it to B; rule two swaps the contents of B and C; rule three moves whatever is in C to D. The marker finishes in",
|
|
"continuations": [
|
|
" Drawer A.",
|
|
" Drawer B.",
|
|
" Drawer C.",
|
|
" Drawer D."
|
|
],
|
|
"tags": [
|
|
"state_update"
|
|
],
|
|
"prediction": 1,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.155341148376465,
|
|
"mean_logprob": -1.0388352870941162,
|
|
"total_sequence_tokens": 45
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -3.6743602752685547,
|
|
"mean_logprob": -0.9185900688171387,
|
|
"total_sequence_tokens": 45
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.7672014236450195,
|
|
"mean_logprob": -1.4418003559112549,
|
|
"total_sequence_tokens": 45
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.906957626342773,
|
|
"mean_logprob": -1.2267394065856934,
|
|
"total_sequence_tokens": 45
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.12024521827697754,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04519589900155552
|
|
},
|
|
{
|
|
"id": "context_tracking-045",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1100,
|
|
"context": "Q is north of R. S is east of Q. T is south of S and level with R. Relative to R, T is",
|
|
"continuations": [
|
|
" east.",
|
|
" west.",
|
|
" north.",
|
|
" at the same position."
|
|
],
|
|
"tags": [
|
|
"spatial"
|
|
],
|
|
"prediction": 3,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.444129943847656,
|
|
"mean_logprob": -5.222064971923828,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.060394287109375,
|
|
"mean_logprob": -4.5301971435546875,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.162068367004395,
|
|
"mean_logprob": -4.581034183502197,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -17.914993286132812,
|
|
"mean_logprob": -3.5829986572265624,
|
|
"total_sequence_tokens": 33
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.9471984863281251,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03621936600029585
|
|
},
|
|
{
|
|
"id": "context_tracking-046",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 2.6999999999999997,
|
|
"item_elo": 1100,
|
|
"context": "There are three switches. Flipping X toggles Y; flipping Y toggles Z. Initially all are off. X is flipped once, then Y is flipped once. At the end, Z is",
|
|
"continuations": [
|
|
" unchanged and off.",
|
|
" on.",
|
|
" removed.",
|
|
" both on and off."
|
|
],
|
|
"tags": [
|
|
"state_update"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -18.077299118041992,
|
|
"mean_logprob": -4.519324779510498,
|
|
"total_sequence_tokens": 47
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -1.6058751344680786,
|
|
"mean_logprob": -0.8029375672340393,
|
|
"total_sequence_tokens": 45
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.36464786529541,
|
|
"mean_logprob": -5.682323932647705,
|
|
"total_sequence_tokens": 45
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -8.00474739074707,
|
|
"mean_logprob": -1.600949478149414,
|
|
"total_sequence_tokens": 48
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7980119109153747,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.045616373001394095
|
|
},
|
|
{
|
|
"id": "context_tracking-047",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 2.6999999999999997,
|
|
"item_elo": 1100,
|
|
"context": "A roster initially reads Arin, Bela, Ciro, Dena. Bela and Dena swap places; then the first and third names swap. The first name becomes",
|
|
"continuations": [
|
|
" Arin.",
|
|
" Dena.",
|
|
" Ciro.",
|
|
" Bela."
|
|
],
|
|
"tags": [
|
|
"permutation"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.308192253112793,
|
|
"mean_logprob": -1.1027307510375977,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.120464324951172,
|
|
"mean_logprob": -1.7068214416503906,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.036705255508423,
|
|
"mean_logprob": -1.0122350851694744,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.030072212219238,
|
|
"mean_logprob": -2.3433574040730796,
|
|
"total_sequence_tokens": 40
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.0904956658681233,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03578504999859433
|
|
},
|
|
{
|
|
"id": "context_tracking-048",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 2.6999999999999997,
|
|
"item_elo": 1100,
|
|
"context": "On odd dates, Gate P leads to Hall 1; on even dates it leads to Hall 2. Gate Q does the opposite. The date is the 18th, and a visitor enters Gate Q, reaching",
|
|
"continuations": [
|
|
" Hall 2.",
|
|
" both halls.",
|
|
" no hall.",
|
|
" Hall 1."
|
|
],
|
|
"tags": [
|
|
"conditional_binding"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -2.4496448040008545,
|
|
"mean_logprob": -0.6124112010002136,
|
|
"total_sequence_tokens": 52
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.987076759338379,
|
|
"mean_logprob": -3.2467691898345947,
|
|
"total_sequence_tokens": 52
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.982742309570312,
|
|
"mean_logprob": -5.9942474365234375,
|
|
"total_sequence_tokens": 51
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -0.7162995934486389,
|
|
"mean_logprob": -0.17907489836215973,
|
|
"total_sequence_tokens": 52
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4333363026380539,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.047340212999188225
|
|
},
|
|
{
|
|
"id": "context_tracking-049",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 2.6999999999999997,
|
|
"item_elo": 1100,
|
|
"context": "Nora is assigned to Vega unless Ishan takes Vega; if he does, Nora takes Lyra. Ishan takes Vega. Nora is therefore assigned to",
|
|
"continuations": [
|
|
" Lyra.",
|
|
" Vega.",
|
|
" both.",
|
|
" neither."
|
|
],
|
|
"tags": [
|
|
"conditional_binding"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -1.1825419664382935,
|
|
"mean_logprob": -0.39418065547943115,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -1.770630121231079,
|
|
"mean_logprob": -0.5902100404103597,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.81666088104248,
|
|
"mean_logprob": -4.90833044052124,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.6995849609375,
|
|
"mean_logprob": -6.84979248046875,
|
|
"total_sequence_tokens": 40
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.19602938493092859,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03791658200134407
|
|
},
|
|
{
|
|
"id": "context_tracking-050",
|
|
"category": "context_tracking",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.2,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.6999999999999997,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1100,
|
|
"context": "The sequence begins amber, cobalt, jade, silver. Remove cobalt, place silver first, then insert rose immediately after amber. The final sequence is",
|
|
"continuations": [
|
|
" silver, rose, amber, jade.",
|
|
" silver, amber, rose, jade.",
|
|
" amber, rose, silver, jade.",
|
|
" silver, amber, jade, rose."
|
|
],
|
|
"tags": [
|
|
"permutation"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 10,
|
|
"sum_logprob": -9.478717803955078,
|
|
"mean_logprob": -0.9478717803955078,
|
|
"total_sequence_tokens": 46
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 10,
|
|
"sum_logprob": -9.959565162658691,
|
|
"mean_logprob": -0.9959565162658691,
|
|
"total_sequence_tokens": 46
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 10,
|
|
"sum_logprob": -11.633020401000977,
|
|
"mean_logprob": -1.1633020401000977,
|
|
"total_sequence_tokens": 46
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 10,
|
|
"sum_logprob": -15.396784782409668,
|
|
"mean_logprob": -1.5396784782409667,
|
|
"total_sequence_tokens": 46
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.04808473587036133,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.044577638000191655
|
|
},
|
|
{
|
|
"id": "quantitative-001",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "Seven plus five equals",
|
|
"continuations": [
|
|
" twelve.",
|
|
" eleven.",
|
|
" thirteen.",
|
|
" fourteen."
|
|
],
|
|
"tags": [
|
|
"arithmetic"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.292641639709473,
|
|
"mean_logprob": -2.6463208198547363,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.131879806518555,
|
|
"mean_logprob": -1.7106266021728516,
|
|
"total_sequence_tokens": 8
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.535145282745361,
|
|
"mean_logprob": -2.1783817609151206,
|
|
"total_sequence_tokens": 8
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.454742431640625,
|
|
"mean_logprob": -2.4849141438802085,
|
|
"total_sequence_tokens": 8
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.46775515874226903,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.033248167999772704
|
|
},
|
|
{
|
|
"id": "quantitative-002",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "A box holds nine pencils. After three are removed, it holds",
|
|
"continuations": [
|
|
" five pencils.",
|
|
" twelve pencils.",
|
|
" seven pencils.",
|
|
" six pencils."
|
|
],
|
|
"tags": [
|
|
"subtraction"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.5479001998901367,
|
|
"mean_logprob": -1.182633399963379,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.5878844261169434,
|
|
"mean_logprob": -1.1959614753723145,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.939874649047852,
|
|
"mean_logprob": -1.6466248830159504,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.488526344299316,
|
|
"mean_logprob": -1.4961754480997722,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.013328075408935547,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02531888900011836
|
|
},
|
|
{
|
|
"id": "quantitative-003",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "Four groups of three objects contain",
|
|
"continuations": [
|
|
" seven objects.",
|
|
" ten objects.",
|
|
" twelve objects.",
|
|
" sixteen objects."
|
|
],
|
|
"tags": [
|
|
"multiplication"
|
|
],
|
|
"prediction": 3,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.508077621459961,
|
|
"mean_logprob": -3.836025873819987,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.469986915588379,
|
|
"mean_logprob": -3.4899956385294595,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -13.069352149963379,
|
|
"mean_logprob": -4.35645071665446,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.693161964416504,
|
|
"mean_logprob": -3.423290491104126,
|
|
"total_sequence_tokens": 10
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.06670514742533351,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028185780000058003
|
|
},
|
|
{
|
|
"id": "quantitative-004",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 1.3,
|
|
"item_elo": 900,
|
|
"context": "Twenty divided equally among five people gives each person",
|
|
"continuations": [
|
|
" two.",
|
|
" three.",
|
|
" five.",
|
|
" four."
|
|
],
|
|
"tags": [
|
|
"division"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.99081802368164,
|
|
"mean_logprob": -4.49540901184082,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.719918251037598,
|
|
"mean_logprob": -4.359959125518799,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -9.545299530029297,
|
|
"mean_logprob": -4.772649765014648,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.664897918701172,
|
|
"mean_logprob": -3.832448959350586,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5275101661682129,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025594724000256974
|
|
},
|
|
{
|
|
"id": "quantitative-005",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 1.3,
|
|
"item_elo": 900,
|
|
"context": "The number immediately after 199 is",
|
|
"continuations": [
|
|
" 200.",
|
|
" 198.",
|
|
" 209.",
|
|
" 299."
|
|
],
|
|
"tags": [
|
|
"number_sense"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -2.5621232986450195,
|
|
"mean_logprob": -0.5124246597290039,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -5.487677574157715,
|
|
"mean_logprob": -1.0975355148315429,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -8.877996444702148,
|
|
"mean_logprob": -1.7755992889404297,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -6.280266761779785,
|
|
"mean_logprob": -1.256053352355957,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.585110855102539,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.024640380001073936
|
|
},
|
|
{
|
|
"id": "quantitative-006",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "Half of eighteen is",
|
|
"continuations": [
|
|
" eight.",
|
|
" nine.",
|
|
" ten.",
|
|
" thirty-six."
|
|
],
|
|
"tags": [
|
|
"fractions"
|
|
],
|
|
"prediction": 3,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.813625812530518,
|
|
"mean_logprob": -2.906812906265259,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.29595947265625,
|
|
"mean_logprob": -3.147979736328125,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.812882423400879,
|
|
"mean_logprob": -3.4064412117004395,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.896859169006348,
|
|
"mean_logprob": -2.224214792251587,
|
|
"total_sequence_tokens": 9
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6825981140136719,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02822259400090843
|
|
},
|
|
{
|
|
"id": "quantitative-007",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "A rectangle with length five and width two has area",
|
|
"continuations": [
|
|
" seven.",
|
|
" twelve.",
|
|
" ten.",
|
|
" fourteen."
|
|
],
|
|
"tags": [
|
|
"geometry"
|
|
],
|
|
"prediction": 3,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.672070503234863,
|
|
"mean_logprob": -3.3360352516174316,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.611666679382324,
|
|
"mean_logprob": -4.305833339691162,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.9228715896606445,
|
|
"mean_logprob": -3.9614357948303223,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.710935592651367,
|
|
"mean_logprob": -2.5703118642171225,
|
|
"total_sequence_tokens": 13
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7657233874003091,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02752333599892154
|
|
},
|
|
{
|
|
"id": "quantitative-008",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 1.3,
|
|
"item_elo": 900,
|
|
"context": "Three quarters of an hour is",
|
|
"continuations": [
|
|
" thirty minutes.",
|
|
" forty minutes.",
|
|
" sixty minutes.",
|
|
" forty-five minutes."
|
|
],
|
|
"tags": [
|
|
"time"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.348872184753418,
|
|
"mean_logprob": -2.782957394917806,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.526144027709961,
|
|
"mean_logprob": -3.175381342569987,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -10.311846733093262,
|
|
"mean_logprob": -2.0623693466186523,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -9.686224937438965,
|
|
"mean_logprob": -1.937244987487793,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.1251243591308593,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03023112299888453
|
|
},
|
|
{
|
|
"id": "quantitative-009",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "A ten-euro item discounted by two euros costs",
|
|
"continuations": [
|
|
" eight euros.",
|
|
" twelve euros.",
|
|
" five euros.",
|
|
" twenty euros."
|
|
],
|
|
"tags": [
|
|
"money"
|
|
],
|
|
"prediction": 2,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -10.715323448181152,
|
|
"mean_logprob": -2.1430646896362306,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -13.067676544189453,
|
|
"mean_logprob": -2.6135353088378905,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -9.866440773010254,
|
|
"mean_logprob": -1.9732881546020509,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -11.636058807373047,
|
|
"mean_logprob": -2.3272117614746093,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.16977653503417978,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02641895899978408
|
|
},
|
|
{
|
|
"id": "quantitative-010",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 1.3,
|
|
"item_elo": 900,
|
|
"context": "The perimeter of a square with side length three is",
|
|
"continuations": [
|
|
" nine.",
|
|
" twelve.",
|
|
" six.",
|
|
" fifteen."
|
|
],
|
|
"tags": [
|
|
"geometry"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.847151756286621,
|
|
"mean_logprob": -5.9235758781433105,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.464617729187012,
|
|
"mean_logprob": -5.732308864593506,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.502902030944824,
|
|
"mean_logprob": -5.751451015472412,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.837128639221191,
|
|
"mean_logprob": -5.918564319610596,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.01914215087890625,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02509022899903357
|
|
},
|
|
{
|
|
"id": "quantitative-011",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 1.3,
|
|
"item_elo": 900,
|
|
"context": "Thirty is five more than",
|
|
"continuations": [
|
|
" twenty.",
|
|
" thirty-five.",
|
|
" twenty-five.",
|
|
" fifteen."
|
|
],
|
|
"tags": [
|
|
"arithmetic"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.370925426483154,
|
|
"mean_logprob": -2.185462713241577,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.961140632629395,
|
|
"mean_logprob": -2.2402851581573486,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.706611156463623,
|
|
"mean_logprob": -1.6766527891159058,
|
|
"total_sequence_tokens": 11
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.271145820617676,
|
|
"mean_logprob": -3.135572910308838,
|
|
"total_sequence_tokens": 9
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5088099241256714,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027746277999540325
|
|
},
|
|
{
|
|
"id": "quantitative-012",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 1.3,
|
|
"item_elo": 900,
|
|
"context": "If one notebook costs four euros, two notebooks cost",
|
|
"continuations": [
|
|
" four euros.",
|
|
" six euros.",
|
|
" twelve euros.",
|
|
" eight euros."
|
|
],
|
|
"tags": [
|
|
"money"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -5.146291255950928,
|
|
"mean_logprob": -1.0292582511901855,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -5.293211936950684,
|
|
"mean_logprob": -1.0586423873901367,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -7.491818904876709,
|
|
"mean_logprob": -1.4983637809753418,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -4.346880912780762,
|
|
"mean_logprob": -0.8693761825561523,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.1598820686340332,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026342607001424767
|
|
},
|
|
{
|
|
"id": "quantitative-013",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "The difference between fifteen and nine is",
|
|
"continuations": [
|
|
" six.",
|
|
" five.",
|
|
" seven.",
|
|
" twenty-four."
|
|
],
|
|
"tags": [
|
|
"subtraction"
|
|
],
|
|
"prediction": 2,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.347989082336426,
|
|
"mean_logprob": -2.673994541168213,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.929468631744385,
|
|
"mean_logprob": -2.4647343158721924,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.034341812133789,
|
|
"mean_logprob": -2.0171709060668945,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.035197257995605,
|
|
"mean_logprob": -2.2587993144989014,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.24162840843200684,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025635102998421644
|
|
},
|
|
{
|
|
"id": "quantitative-014",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "Counting by tens, the number after forty is",
|
|
"continuations": [
|
|
" forty-one.",
|
|
" fifty.",
|
|
" sixty.",
|
|
" thirty."
|
|
],
|
|
"tags": [
|
|
"number_sense"
|
|
],
|
|
"prediction": 3,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.080322265625,
|
|
"mean_logprob": -1.52008056640625,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.013950347900391,
|
|
"mean_logprob": -2.0069751739501953,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.232993125915527,
|
|
"mean_logprob": -1.8082482814788818,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -2.881405830383301,
|
|
"mean_logprob": -1.4407029151916504,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.07937765121459961,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025576610998541582
|
|
},
|
|
{
|
|
"id": "quantitative-015",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "Two meters equal",
|
|
"continuations": [
|
|
" twenty centimeters.",
|
|
" two hundred millimeters.",
|
|
" two hundred centimeters.",
|
|
" two thousand centimeters."
|
|
],
|
|
"tags": [
|
|
"units"
|
|
],
|
|
"prediction": 1,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.974717140197754,
|
|
"mean_logprob": -4.991572380065918,
|
|
"total_sequence_tokens": 6
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -16.25657081604004,
|
|
"mean_logprob": -3.251314163208008,
|
|
"total_sequence_tokens": 8
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.64132308959961,
|
|
"mean_logprob": -3.9103307723999023,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.949125289916992,
|
|
"mean_logprob": -4.487281322479248,
|
|
"total_sequence_tokens": 7
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6590166091918945,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027211262000491843
|
|
},
|
|
{
|
|
"id": "quantitative-016",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "The average of four and eight is",
|
|
"continuations": [
|
|
" four.",
|
|
" five.",
|
|
" eight.",
|
|
" six."
|
|
],
|
|
"tags": [
|
|
"average"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.802121162414551,
|
|
"mean_logprob": -2.9010605812072754,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.14479398727417,
|
|
"mean_logprob": -3.072396993637085,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.672871112823486,
|
|
"mean_logprob": -3.336435556411743,
|
|
"total_sequence_tokens": 9
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.14555025100708,
|
|
"mean_logprob": -3.07277512550354,
|
|
"total_sequence_tokens": 9
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.17133641242980957,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02484983700014709
|
|
},
|
|
{
|
|
"id": "quantitative-017",
|
|
"category": "quantitative",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.3,
|
|
"earned_points": 0.0,
|
|
"item_elo": 900,
|
|
"context": "A dozen eggs contains",
|
|
"continuations": [
|
|
" twelve eggs.",
|
|
" ten eggs.",
|
|
" six eggs.",
|
|
" twenty eggs."
|
|
],
|
|
"tags": [
|
|
"number_sense"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.102995872497559,
|
|
"mean_logprob": -3.034331957499186,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.773576259613037,
|
|
"mean_logprob": -2.591192086537679,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.962491035461426,
|
|
"mean_logprob": -2.987497011820475,
|
|
"total_sequence_tokens": 7
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -8.10993480682373,
|
|
"mean_logprob": -2.7033116022745767,
|
|
"total_sequence_tokens": 7
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.11211951573689749,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.024494163000781555
|
|
},
|
|
{
|
|
"id": "quantitative-018",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "A train travels 180 kilometers in three hours at a constant rate. Its speed is",
|
|
"continuations": [
|
|
" forty kilometers per hour.",
|
|
" sixty kilometers per hour.",
|
|
" ninety kilometers per hour.",
|
|
" one hundred eighty kilometers per hour."
|
|
],
|
|
"tags": [
|
|
"rate"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -16.961780548095703,
|
|
"mean_logprob": -3.392356109619141,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -26.406064987182617,
|
|
"mean_logprob": -3.7722949981689453,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.92763900756836,
|
|
"mean_logprob": -2.56109128679548,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -23.113128662109375,
|
|
"mean_logprob": -2.889141082763672,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.32804979596819184,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029687533000469557
|
|
},
|
|
{
|
|
"id": "quantitative-019",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "A jacket priced at 80 euros is reduced by 25 percent. The sale price is",
|
|
"continuations": [
|
|
" fifty euros.",
|
|
" fifty-five euros.",
|
|
" sixty euros.",
|
|
" sixty-five euros."
|
|
],
|
|
"tags": [
|
|
"percent"
|
|
],
|
|
"prediction": 3,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -18.02532386779785,
|
|
"mean_logprob": -3.6050647735595702,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -19.415939331054688,
|
|
"mean_logprob": -2.773705618722098,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.058008193969727,
|
|
"mean_logprob": -2.4368583134242465,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 9,
|
|
"sum_logprob": -13.648637771606445,
|
|
"mean_logprob": -1.5165153079562717,
|
|
"total_sequence_tokens": 33
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.9203430054679749,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04079642699980468
|
|
},
|
|
{
|
|
"id": "quantitative-020",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 1.9500000000000002,
|
|
"item_elo": 1000,
|
|
"context": "A tank contains 36 liters. One third is drained, leaving",
|
|
"continuations": [
|
|
" twelve liters.",
|
|
" eighteen liters.",
|
|
" thirty liters.",
|
|
" twenty-four liters."
|
|
],
|
|
"tags": [
|
|
"fractions"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.342558860778809,
|
|
"mean_logprob": -4.7808529535929365,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.174556732177734,
|
|
"mean_logprob": -4.043639183044434,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.89756202697754,
|
|
"mean_logprob": -5.63252067565918,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -15.412789344787598,
|
|
"mean_logprob": -3.0825578689575197,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.9610813140869139,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.045352904999163
|
|
},
|
|
{
|
|
"id": "quantitative-021",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "Five consecutive integers begin with 12. Their sum is",
|
|
"continuations": [
|
|
" seventy.",
|
|
" sixty.",
|
|
" seventy-five.",
|
|
" eighty."
|
|
],
|
|
"tags": [
|
|
"sequence"
|
|
],
|
|
"prediction": 2,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.021854400634766,
|
|
"mean_logprob": -5.673951466878255,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -23.48359489440918,
|
|
"mean_logprob": -5.870898723602295,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -14.351556777954102,
|
|
"mean_logprob": -2.8703113555908204,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.526702880859375,
|
|
"mean_logprob": -5.842234293619792,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.8036401112874345,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.031220798000504146
|
|
},
|
|
{
|
|
"id": "quantitative-022",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "A right triangle has legs of length 6 and 8. Its hypotenuse has length",
|
|
"continuations": [
|
|
" nine.",
|
|
" ten.",
|
|
" twelve.",
|
|
" fourteen."
|
|
],
|
|
"tags": [
|
|
"geometry"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.915306091308594,
|
|
"mean_logprob": -5.957653045654297,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.793683052062988,
|
|
"mean_logprob": -6.896841526031494,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.201858520507812,
|
|
"mean_logprob": -6.600929260253906,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -18.208065032958984,
|
|
"mean_logprob": -6.069355010986328,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.11170196533203125,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027931208000154584
|
|
},
|
|
{
|
|
"id": "quantitative-023",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "If x plus 7 equals 19, then x equals",
|
|
"continuations": [
|
|
" ten.",
|
|
" eleven.",
|
|
" twelve.",
|
|
" twenty-six."
|
|
],
|
|
"tags": [
|
|
"algebra"
|
|
],
|
|
"prediction": 3,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.407376289367676,
|
|
"mean_logprob": -5.703688144683838,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -13.684794425964355,
|
|
"mean_logprob": -4.561598141988118,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.927817344665527,
|
|
"mean_logprob": -6.963908672332764,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.525569915771484,
|
|
"mean_logprob": -3.881392478942871,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6802056630452471,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03209438700105238
|
|
},
|
|
{
|
|
"id": "quantitative-024",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 1.9500000000000002,
|
|
"item_elo": 1000,
|
|
"context": "A worker earns 18 euros per hour for seven hours. Total earnings are",
|
|
"continuations": [
|
|
" 108 euros.",
|
|
" 116 euros.",
|
|
" 136 euros.",
|
|
" 126 euros."
|
|
],
|
|
"tags": [
|
|
"multiplication"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -9.774751663208008,
|
|
"mean_logprob": -1.221843957901001,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -10.015142440795898,
|
|
"mean_logprob": -1.2518928050994873,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 8,
|
|
"sum_logprob": -11.240349769592285,
|
|
"mean_logprob": -1.4050437211990356,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -8.623146057128906,
|
|
"mean_logprob": -1.0778932571411133,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.1439507007598877,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.06568012900061149
|
|
},
|
|
{
|
|
"id": "quantitative-025",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 1.9500000000000002,
|
|
"item_elo": 1000,
|
|
"context": "The ratio of red to blue marbles is 3 to 5. If there are 15 red marbles, there are",
|
|
"continuations": [
|
|
" twenty-five blue marbles.",
|
|
" nine blue marbles.",
|
|
" eighteen blue marbles.",
|
|
" seventy-five blue marbles."
|
|
],
|
|
"tags": [
|
|
"ratio"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -16.056371688842773,
|
|
"mean_logprob": -2.6760619481404624,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.535652160644531,
|
|
"mean_logprob": -3.133913040161133,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -17.886301040649414,
|
|
"mean_logprob": -3.577260208129883,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -20.623779296875,
|
|
"mean_logprob": -2.946254185267857,
|
|
"total_sequence_tokens": 32
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.2701922371273948,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04084540600160835
|
|
},
|
|
{
|
|
"id": "quantitative-026",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "A circle has radius 4. Using pi as 3.14, its circumference is approximately",
|
|
"continuations": [
|
|
" 12.56.",
|
|
" 25.12.",
|
|
" 50.24.",
|
|
" 16.00."
|
|
],
|
|
"tags": [
|
|
"geometry"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -3.247330665588379,
|
|
"mean_logprob": -0.46390438079833984,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -11.988222122192383,
|
|
"mean_logprob": -1.7126031603131975,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -10.607452392578125,
|
|
"mean_logprob": -1.515350341796875,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -26.597305297851562,
|
|
"mean_logprob": -3.799615042550223,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.0514459609985352,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026777819999551866
|
|
},
|
|
{
|
|
"id": "quantitative-027",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 1.9500000000000002,
|
|
"item_elo": 1000,
|
|
"context": "An account balance rises from 240 to 300. The percentage increase is",
|
|
"continuations": [
|
|
" fifteen percent.",
|
|
" twenty percent.",
|
|
" twenty-five percent.",
|
|
" sixty percent."
|
|
],
|
|
"tags": [
|
|
"percent"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -19.058584213256836,
|
|
"mean_logprob": -6.352861404418945,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -19.500022888183594,
|
|
"mean_logprob": -6.500007629394531,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -18.61857795715332,
|
|
"mean_logprob": -3.723715591430664,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -19.820444107055664,
|
|
"mean_logprob": -3.9640888214111327,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.24037322998046884,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02628384899981029
|
|
},
|
|
{
|
|
"id": "quantitative-028",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "A bus leaves at 09:35 and arrives at 11:05. The journey lasts",
|
|
"continuations": [
|
|
" seventy minutes.",
|
|
" eighty minutes.",
|
|
" one hundred minutes.",
|
|
" ninety minutes."
|
|
],
|
|
"tags": [
|
|
"time"
|
|
],
|
|
"prediction": 2,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -19.287830352783203,
|
|
"mean_logprob": -4.821957588195801,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -33.98118591308594,
|
|
"mean_logprob": -8.495296478271484,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -19.279085159301758,
|
|
"mean_logprob": -4.8197712898254395,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -30.71573829650879,
|
|
"mean_logprob": -6.143147659301758,
|
|
"total_sequence_tokens": 30
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.002186298370361328,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02756844000032288
|
|
},
|
|
{
|
|
"id": "quantitative-029",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 1.9500000000000002,
|
|
"item_elo": 1000,
|
|
"context": "The median of 3, 7, 8, 12, and 20 is",
|
|
"continuations": [
|
|
" eight.",
|
|
" seven.",
|
|
" ten.",
|
|
" twelve."
|
|
],
|
|
"tags": [
|
|
"statistics"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -12.059478759765625,
|
|
"mean_logprob": -6.0297393798828125,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -12.399323463439941,
|
|
"mean_logprob": -6.199661731719971,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -14.319886207580566,
|
|
"mean_logprob": -7.159943103790283,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -14.025663375854492,
|
|
"mean_logprob": -7.012831687927246,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.1699223518371582,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05327712399957818
|
|
},
|
|
{
|
|
"id": "quantitative-030",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "A 48-page booklet is printed with four pages per sheet. It requires",
|
|
"continuations": [
|
|
" ten sheets.",
|
|
" twelve sheets.",
|
|
" sixteen sheets.",
|
|
" forty-four sheets."
|
|
],
|
|
"tags": [
|
|
"division"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.124528884887695,
|
|
"mean_logprob": -3.708176294962565,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.980240821838379,
|
|
"mean_logprob": -3.6600802739461265,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.329124450683594,
|
|
"mean_logprob": -3.0822811126708984,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -16.06705093383789,
|
|
"mean_logprob": -3.213410186767578,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.1311290740966795,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05290328900082386
|
|
},
|
|
{
|
|
"id": "quantitative-031",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "Three fifths written as a decimal is",
|
|
"continuations": [
|
|
" 0.3.",
|
|
" 0.5.",
|
|
" 0.6.",
|
|
" 0.8."
|
|
],
|
|
"tags": [
|
|
"fractions"
|
|
],
|
|
"prediction": 3,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -7.394290924072266,
|
|
"mean_logprob": -1.478858184814453,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -4.609033107757568,
|
|
"mean_logprob": -0.9218066215515137,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -5.725163459777832,
|
|
"mean_logprob": -1.1450326919555665,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -4.468921661376953,
|
|
"mean_logprob": -0.8937843322753907,
|
|
"total_sequence_tokens": 13
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.028022289276123047,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025549125999532407
|
|
},
|
|
{
|
|
"id": "quantitative-032",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 1.9500000000000002,
|
|
"item_elo": 1000,
|
|
"context": "A shop buys an item for 30 euros and sells it for 42 euros. The profit is",
|
|
"continuations": [
|
|
" 10 euros.",
|
|
" 11 euros.",
|
|
" 72 euros.",
|
|
" 12 euros."
|
|
],
|
|
"tags": [
|
|
"money"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -3.9913277626037598,
|
|
"mean_logprob": -0.5701896803719657,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -6.558568954467773,
|
|
"mean_logprob": -0.9369384220668248,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -8.563762664794922,
|
|
"mean_logprob": -1.2233946663992745,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -3.681147336959839,
|
|
"mean_logprob": -0.5258781909942627,
|
|
"total_sequence_tokens": 33
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.04431148937770302,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03577348799990432
|
|
},
|
|
{
|
|
"id": "quantitative-033",
|
|
"category": "quantitative",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 1.9500000000000002,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1000,
|
|
"context": "If 2x minus 4 equals 16, then x equals",
|
|
"continuations": [
|
|
" ten.",
|
|
" six.",
|
|
" eight.",
|
|
" twelve."
|
|
],
|
|
"tags": [
|
|
"algebra"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.682738304138184,
|
|
"mean_logprob": -5.341369152069092,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.106553077697754,
|
|
"mean_logprob": -5.053276538848877,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.52470874786377,
|
|
"mean_logprob": -5.262354373931885,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.924354553222656,
|
|
"mean_logprob": -5.462177276611328,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.2090778350830078,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03075260699915816
|
|
},
|
|
{
|
|
"id": "quantitative-034",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A price increases by 20 percent and is then reduced by 20 percent. Starting from 100 euros, the final price is",
|
|
"continuations": [
|
|
" 100 euros.",
|
|
" 96 euros.",
|
|
" 80 euros.",
|
|
" 104 euros."
|
|
],
|
|
"tags": [
|
|
"percent"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -14.256305694580078,
|
|
"mean_logprob": -1.7820382118225098,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -10.194211959838867,
|
|
"mean_logprob": -1.4563159942626953,
|
|
"total_sequence_tokens": 39
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -7.444924831390381,
|
|
"mean_logprob": -1.0635606901986259,
|
|
"total_sequence_tokens": 39
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -11.327791213989258,
|
|
"mean_logprob": -1.4159739017486572,
|
|
"total_sequence_tokens": 40
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.35241321155003136,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0364853069986566
|
|
},
|
|
{
|
|
"id": "quantitative-035",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 2.9250000000000003,
|
|
"item_elo": 1150,
|
|
"context": "A mixture contains milk and water in a 5 to 2 ratio. In 21 liters of mixture, the amount of water is",
|
|
"continuations": [
|
|
" five liters.",
|
|
" seven liters.",
|
|
" six liters.",
|
|
" fifteen liters."
|
|
],
|
|
"tags": [
|
|
"ratio"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.469293594360352,
|
|
"mean_logprob": -4.823097864786784,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.154003143310547,
|
|
"mean_logprob": -5.384667714436849,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -13.7432279586792,
|
|
"mean_logprob": -4.581075986226399,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.343059539794922,
|
|
"mean_logprob": -5.114353179931641,
|
|
"total_sequence_tokens": 31
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.2420218785603847,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.036742966000019806
|
|
},
|
|
{
|
|
"id": "quantitative-036",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 2.9250000000000003,
|
|
"item_elo": 1150,
|
|
"context": "Two numbers have sum 46 and difference 8. The larger number is",
|
|
"continuations": [
|
|
" nineteen.",
|
|
" twenty-three.",
|
|
" thirty-five.",
|
|
" twenty-seven."
|
|
],
|
|
"tags": [
|
|
"algebra"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -20.145109176635742,
|
|
"mean_logprob": -5.0362772941589355,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.645545959472656,
|
|
"mean_logprob": -3.911386489868164,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.99804973602295,
|
|
"mean_logprob": -3.7495124340057373,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.937511444091797,
|
|
"mean_logprob": -3.734377861022949,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.015134572982788086,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027836683999339584
|
|
},
|
|
{
|
|
"id": "quantitative-037",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A car travels the first 120 kilometers at 60 km/h and the next 120 at 40 km/h. Its average speed for the full trip is",
|
|
"continuations": [
|
|
" 48 km/h.",
|
|
" 50 km/h.",
|
|
" 45 km/h.",
|
|
" 52 km/h."
|
|
],
|
|
"tags": [
|
|
"rate"
|
|
],
|
|
"prediction": 2,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -6.520872116088867,
|
|
"mean_logprob": -0.9315531594412667,
|
|
"total_sequence_tokens": 47
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -3.3077588081359863,
|
|
"mean_logprob": -0.4725369725908552,
|
|
"total_sequence_tokens": 47
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -0.6295626163482666,
|
|
"mean_logprob": -0.08993751662118095,
|
|
"total_sequence_tokens": 47
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -4.594418048858643,
|
|
"mean_logprob": -0.6563454355512347,
|
|
"total_sequence_tokens": 47
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.38259945596967426,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03457150299982459
|
|
},
|
|
{
|
|
"id": "quantitative-038",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A rectangle's length is three more than its width. If its perimeter is 30, its width is",
|
|
"continuations": [
|
|
" five.",
|
|
" six.",
|
|
" seven.",
|
|
" nine."
|
|
],
|
|
"tags": [
|
|
"algebra"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.429022789001465,
|
|
"mean_logprob": -5.714511394500732,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.57111930847168,
|
|
"mean_logprob": -5.78555965423584,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -12.157730102539062,
|
|
"mean_logprob": -6.078865051269531,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.113155364990234,
|
|
"mean_logprob": -6.556577682495117,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.07104825973510742,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.031772250000358326
|
|
},
|
|
{
|
|
"id": "quantitative-039",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A bag has four red, three blue, and five green balls. The probability of drawing a blue ball is",
|
|
"continuations": [
|
|
" one third.",
|
|
" three fifths.",
|
|
" one quarter.",
|
|
" three quarters."
|
|
],
|
|
"tags": [
|
|
"probability"
|
|
],
|
|
"prediction": 1,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.496288299560547,
|
|
"mean_logprob": -5.832096099853516,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.568428039550781,
|
|
"mean_logprob": -3.8921070098876953,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.224609375,
|
|
"mean_logprob": -5.408203125,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.390769958496094,
|
|
"mean_logprob": -5.796923319498698,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.5160961151123047,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03168500199899427
|
|
},
|
|
{
|
|
"id": "quantitative-040",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "An investment of 2,000 euros earns simple interest at 4 percent per year for three years. The interest earned is",
|
|
"continuations": [
|
|
" 80 euros.",
|
|
" 160 euros.",
|
|
" 2,240 euros.",
|
|
" 240 euros."
|
|
],
|
|
"tags": [
|
|
"finance"
|
|
],
|
|
"prediction": 1,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -8.198927879333496,
|
|
"mean_logprob": -1.1712754113333566,
|
|
"total_sequence_tokens": 37
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -8.330878257751465,
|
|
"mean_logprob": -1.041359782218933,
|
|
"total_sequence_tokens": 38
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 10,
|
|
"sum_logprob": -17.7349910736084,
|
|
"mean_logprob": -1.77349910736084,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -8.385602951049805,
|
|
"mean_logprob": -1.0482003688812256,
|
|
"total_sequence_tokens": 38
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.0068405866622924805,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.036835486000200035
|
|
},
|
|
{
|
|
"id": "quantitative-041",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "The sequence 2, 6, 12, 20, 30 continues with",
|
|
"continuations": [
|
|
" 42.",
|
|
" 36.",
|
|
" 40.",
|
|
" 44."
|
|
],
|
|
"tags": [
|
|
"sequence"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.798002243041992,
|
|
"mean_logprob": -3.199500560760498,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.089204788208008,
|
|
"mean_logprob": -2.772301197052002,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.132857322692871,
|
|
"mean_logprob": -3.0332143306732178,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.851228713989258,
|
|
"mean_logprob": -3.2128071784973145,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.2609131336212158,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.030952609000451048
|
|
},
|
|
{
|
|
"id": "quantitative-042",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A machine completes three jobs in 18 minutes at a constant rate. At that rate, eight jobs take",
|
|
"continuations": [
|
|
" 36 minutes.",
|
|
" 48 minutes.",
|
|
" 54 minutes.",
|
|
" 144 minutes."
|
|
],
|
|
"tags": [
|
|
"rate"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -4.16306209564209,
|
|
"mean_logprob": -0.832612419128418,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -4.029392242431641,
|
|
"mean_logprob": -0.8058784484863282,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -3.299407720565796,
|
|
"mean_logprob": -0.6598815441131591,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -7.349918842315674,
|
|
"mean_logprob": -1.224986473719279,
|
|
"total_sequence_tokens": 28
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.14599690437316903,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027485315999001614
|
|
},
|
|
{
|
|
"id": "quantitative-043",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 2.9250000000000003,
|
|
"item_elo": 1150,
|
|
"context": "The mean of six numbers is 14. Five of them sum to 61. The sixth number is",
|
|
"continuations": [
|
|
" fourteen.",
|
|
" seventeen.",
|
|
" twenty-three.",
|
|
" eighty-four."
|
|
],
|
|
"tags": [
|
|
"statistics"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -16.144824981689453,
|
|
"mean_logprob": -5.381608327229817,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.930179595947266,
|
|
"mean_logprob": -3.976726531982422,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.954082489013672,
|
|
"mean_logprob": -3.488520622253418,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -19.000680923461914,
|
|
"mean_logprob": -3.800136184692383,
|
|
"total_sequence_tokens": 28
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.31161556243896493,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025930818999768235
|
|
},
|
|
{
|
|
"id": "quantitative-044",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A map scale is 1 to 50,000. A distance of 6 centimeters on the map represents",
|
|
"continuations": [
|
|
" 300 meters.",
|
|
" 30 kilometers.",
|
|
" 300 kilometers.",
|
|
" 3 kilometers."
|
|
],
|
|
"tags": [
|
|
"scale"
|
|
],
|
|
"prediction": 2,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -9.360454559326172,
|
|
"mean_logprob": -1.5600757598876953,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -9.09926986694336,
|
|
"mean_logprob": -1.8198539733886718,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -8.061050415039062,
|
|
"mean_logprob": -1.3435084025065105,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.9915056228637695,
|
|
"mean_logprob": -1.9978764057159424,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.21656735738118482,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0256035399997927
|
|
},
|
|
{
|
|
"id": "quantitative-045",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 2.9250000000000003,
|
|
"item_elo": 1150,
|
|
"context": "If three quarters of a number is 27, the number is",
|
|
"continuations": [
|
|
" thirty-six.",
|
|
" twenty-one.",
|
|
" thirty.",
|
|
" forty."
|
|
],
|
|
"tags": [
|
|
"algebra"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -13.211942672729492,
|
|
"mean_logprob": -3.302985668182373,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.152857780456543,
|
|
"mean_logprob": -3.7882144451141357,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -15.172837257385254,
|
|
"mean_logprob": -7.586418628692627,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -13.899062156677246,
|
|
"mean_logprob": -6.949531078338623,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4852287769317627,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02752245900046546
|
|
},
|
|
{
|
|
"id": "quantitative-046",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A 15-liter solution is 40 percent salt. Adding 5 liters of water makes the new salt concentration",
|
|
"continuations": [
|
|
" twenty percent.",
|
|
" thirty percent.",
|
|
" thirty-five percent.",
|
|
" forty-five percent."
|
|
],
|
|
"tags": [
|
|
"mixture"
|
|
],
|
|
"prediction": 3,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -20.30893325805664,
|
|
"mean_logprob": -6.769644419352214,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -20.39630126953125,
|
|
"mean_logprob": -6.79876708984375,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -17.16763687133789,
|
|
"mean_logprob": -3.433527374267578,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -16.944379806518555,
|
|
"mean_logprob": -3.388875961303711,
|
|
"total_sequence_tokens": 30
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.0446514129638671,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029969453000376234
|
|
},
|
|
{
|
|
"id": "quantitative-047",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A square's diagonal has length 10. Its area is",
|
|
"continuations": [
|
|
" twenty-five.",
|
|
" forty.",
|
|
" fifty.",
|
|
" one hundred."
|
|
],
|
|
"tags": [
|
|
"geometry"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.12101936340332,
|
|
"mean_logprob": -4.28025484085083,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -21.154712677001953,
|
|
"mean_logprob": -10.577356338500977,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -21.690587997436523,
|
|
"mean_logprob": -10.845293998718262,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -22.510478973388672,
|
|
"mean_logprob": -7.503492991129558,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.2232381502787275,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03216763900127262
|
|
},
|
|
{
|
|
"id": "quantitative-048",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A clock gains two minutes every three hours. After 24 real hours, it will be ahead by",
|
|
"continuations": [
|
|
" eight minutes.",
|
|
" twelve minutes.",
|
|
" eighteen minutes.",
|
|
" sixteen minutes."
|
|
],
|
|
"tags": [
|
|
"rate"
|
|
],
|
|
"prediction": 1,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.064818382263184,
|
|
"mean_logprob": -2.021606127421061,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.02801513671875,
|
|
"mean_logprob": -2.00933837890625,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.82884693145752,
|
|
"mean_logprob": -2.45721173286438,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.988325119018555,
|
|
"mean_logprob": -2.2470812797546387,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.01226774851481105,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.031557569000142394
|
|
},
|
|
{
|
|
"id": "quantitative-049",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1150,
|
|
"context": "A number is increased by 8, doubled, and then reduced by 6 to give 30. The original number is",
|
|
"continuations": [
|
|
" ten.",
|
|
" eight.",
|
|
" twelve.",
|
|
" fourteen."
|
|
],
|
|
"tags": [
|
|
"algebra"
|
|
],
|
|
"prediction": 3,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.160680770874023,
|
|
"mean_logprob": -5.580340385437012,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.79175853729248,
|
|
"mean_logprob": -4.39587926864624,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.474509239196777,
|
|
"mean_logprob": -5.737254619598389,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -10.697616577148438,
|
|
"mean_logprob": -3.5658721923828125,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8300070762634277,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03413807600009022
|
|
},
|
|
{
|
|
"id": "quantitative-050",
|
|
"category": "quantitative",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.3,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 2.9250000000000003,
|
|
"earned_points": 2.9250000000000003,
|
|
"item_elo": 1150,
|
|
"context": "A store marks an item up 25 percent from a cost of 64 euros. The marked price is",
|
|
"continuations": [
|
|
" 72 euros.",
|
|
" 80 euros.",
|
|
" 84 euros.",
|
|
" 89 euros."
|
|
],
|
|
"tags": [
|
|
"percent"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -4.643354415893555,
|
|
"mean_logprob": -0.6633363451276507,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -4.039778709411621,
|
|
"mean_logprob": -0.5771112442016602,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -6.562363147735596,
|
|
"mean_logprob": -0.9374804496765137,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -8.166106224060059,
|
|
"mean_logprob": -1.1665866034371513,
|
|
"total_sequence_tokens": 32
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.08622510092599056,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027850759999637376
|
|
},
|
|
{
|
|
"id": "logical_reasoning-001",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "All robins are birds. Pip is a robin. Therefore, Pip is",
|
|
"continuations": [
|
|
" a bird.",
|
|
" not a bird.",
|
|
" a fish.",
|
|
" necessarily blue."
|
|
],
|
|
"tags": [
|
|
"deduction"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.120014190673828,
|
|
"mean_logprob": -1.7066713968912761,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.987151145935059,
|
|
"mean_logprob": -1.4967877864837646,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.111690044403076,
|
|
"mean_logprob": -1.7038966814676921,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.19256591796875,
|
|
"mean_logprob": -5.730855305989583,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.20710889498392748,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028051785000570817
|
|
},
|
|
{
|
|
"id": "logical_reasoning-002",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "No squares are circles. Shape K is a square. Shape K is therefore",
|
|
"continuations": [
|
|
" a circle.",
|
|
" not a square.",
|
|
" both shapes.",
|
|
" not a circle."
|
|
],
|
|
"tags": [
|
|
"deduction"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.540952205657959,
|
|
"mean_logprob": -0.846984068552653,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.215645790100098,
|
|
"mean_logprob": -1.3039114475250244,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.818357467651367,
|
|
"mean_logprob": -4.939452489217122,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.576274871826172,
|
|
"mean_logprob": -1.644068717956543,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.45692737897237146,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026913691999652656
|
|
},
|
|
{
|
|
"id": "logical_reasoning-003",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "If the switch is on, the lamp glows. The switch is on, so the lamp",
|
|
"continuations": [
|
|
" must be broken.",
|
|
" is off.",
|
|
" glows.",
|
|
" has no switch."
|
|
],
|
|
"tags": [
|
|
"modus_ponens"
|
|
],
|
|
"prediction": 1,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.94361686706543,
|
|
"mean_logprob": -2.9859042167663574,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.789653778076172,
|
|
"mean_logprob": -0.9298845926920573,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.650395154953003,
|
|
"mean_logprob": -1.2167983849843342,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -19.535369873046875,
|
|
"mean_logprob": -4.883842468261719,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.286913792292277,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027348768000592827
|
|
},
|
|
{
|
|
"id": "logical_reasoning-004",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "A is taller than B, and B is taller than C. It follows that",
|
|
"continuations": [
|
|
" C is taller than A.",
|
|
" A and C are equal.",
|
|
" B is shortest.",
|
|
" A is taller than C."
|
|
],
|
|
"tags": [
|
|
"transitive"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -3.1261911392211914,
|
|
"mean_logprob": -0.4465987341744559,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -10.728402137756348,
|
|
"mean_logprob": -1.7880670229593914,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.119539260864258,
|
|
"mean_logprob": -3.5298848152160645,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -5.211345195770264,
|
|
"mean_logprob": -0.7444778851100377,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.29787915093558176,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027715070000340347
|
|
},
|
|
{
|
|
"id": "logical_reasoning-005",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 1.35,
|
|
"item_elo": 950,
|
|
"context": "Either the coin is in the cup or in the box. It is not in the cup. Thus it is",
|
|
"continuations": [
|
|
" in the box.",
|
|
" in both places.",
|
|
" nowhere.",
|
|
" still in the cup."
|
|
],
|
|
"tags": [
|
|
"disjunction"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.2625555992126465,
|
|
"mean_logprob": -1.0656388998031616,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.732442855834961,
|
|
"mean_logprob": -2.9331107139587402,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -21.61458969116211,
|
|
"mean_logprob": -7.20486323038737,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -8.50075912475586,
|
|
"mean_logprob": -1.7001518249511718,
|
|
"total_sequence_tokens": 28
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6345129251480102,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027197057999728713
|
|
},
|
|
{
|
|
"id": "logical_reasoning-006",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 1.35,
|
|
"item_elo": 950,
|
|
"context": "Every metal key conducts electricity. This object is a metal key, so it",
|
|
"continuations": [
|
|
" cannot be metal.",
|
|
" conducts electricity.",
|
|
" must be wooden.",
|
|
" proves all objects conduct."
|
|
],
|
|
"tags": [
|
|
"deduction"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.674589157104492,
|
|
"mean_logprob": -4.168647289276123,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.755519866943359,
|
|
"mean_logprob": -1.1888799667358398,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -14.105069160461426,
|
|
"mean_logprob": -3.5262672901153564,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -37.62774658203125,
|
|
"mean_logprob": -7.52554931640625,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.3373873233795166,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027393565998863778
|
|
},
|
|
{
|
|
"id": "logical_reasoning-007",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 1.35,
|
|
"item_elo": 950,
|
|
"context": "If today is Monday, tomorrow is Tuesday. Today is Monday. Tomorrow is",
|
|
"continuations": [
|
|
" Sunday.",
|
|
" Wednesday.",
|
|
" Tuesday.",
|
|
" Monday."
|
|
],
|
|
"tags": [
|
|
"conditional"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.646546363830566,
|
|
"mean_logprob": -2.323273181915283,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -2.154924154281616,
|
|
"mean_logprob": -1.077462077140808,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -0.5209131240844727,
|
|
"mean_logprob": -0.26045656204223633,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.987026214599609,
|
|
"mean_logprob": -2.4935131072998047,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8170055150985718,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03134048799984157
|
|
},
|
|
{
|
|
"id": "logical_reasoning-008",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "No red token is large. Token P is red. Token P is",
|
|
"continuations": [
|
|
" definitely large.",
|
|
" both red and large.",
|
|
" not red.",
|
|
" not large."
|
|
],
|
|
"tags": [
|
|
"deduction"
|
|
],
|
|
"prediction": 2,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.585792541503906,
|
|
"mean_logprob": -4.861930847167969,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 5,
|
|
"sum_logprob": -10.64543342590332,
|
|
"mean_logprob": -2.1290866851806642,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.6387014389038086,
|
|
"mean_logprob": -0.8795671463012695,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.280209541320801,
|
|
"mean_logprob": -2.093403180440267,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.2138360341389975,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027026356001442764
|
|
},
|
|
{
|
|
"id": "logical_reasoning-009",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 1.35,
|
|
"item_elo": 950,
|
|
"context": "Only one of Door A and Door B is open. Door A is closed, so",
|
|
"continuations": [
|
|
" Door B is open.",
|
|
" both are closed.",
|
|
" both are open.",
|
|
" Door B must be closed."
|
|
],
|
|
"tags": [
|
|
"exclusive_or"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -2.959444761276245,
|
|
"mean_logprob": -0.49324079354604083,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -12.22856330871582,
|
|
"mean_logprob": -3.057140827178955,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.97695541381836,
|
|
"mean_logprob": -2.74423885345459,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -4.827787399291992,
|
|
"mean_logprob": -0.6896839141845703,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.19644312063852948,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02870630799952778
|
|
},
|
|
{
|
|
"id": "logical_reasoning-010",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 1.35,
|
|
"item_elo": 950,
|
|
"context": "All cedar trees are plants. No plant is a machine. Therefore, no cedar tree is",
|
|
"continuations": [
|
|
" a plant.",
|
|
" a machine.",
|
|
" a tree.",
|
|
" alive."
|
|
],
|
|
"tags": [
|
|
"syllogism"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.419729232788086,
|
|
"mean_logprob": -2.473243077596029,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -0.6436737775802612,
|
|
"mean_logprob": -0.21455792586008707,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -11.101336479187012,
|
|
"mean_logprob": -3.7004454930623374,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.84523868560791,
|
|
"mean_logprob": -3.922619342803955,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.258685151735942,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026393262000055984
|
|
},
|
|
{
|
|
"id": "logical_reasoning-011",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 1.35,
|
|
"item_elo": 950,
|
|
"context": "If it rains, the path gets wet. The path is not wet. Therefore, it",
|
|
"continuations": [
|
|
" certainly snowed.",
|
|
" must be night.",
|
|
" did not rain.",
|
|
" is still raining."
|
|
],
|
|
"tags": [
|
|
"modus_tollens"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -18.82676887512207,
|
|
"mean_logprob": -4.706692218780518,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.50233268737793,
|
|
"mean_logprob": -2.6255831718444824,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.35506010055542,
|
|
"mean_logprob": -1.588765025138855,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -9.369444847106934,
|
|
"mean_logprob": -1.8738889694213867,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.28512394428253174,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02827742499903252
|
|
},
|
|
{
|
|
"id": "logical_reasoning-012",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "L is before M, and M is before N. In this ordering, N is",
|
|
"continuations": [
|
|
" first.",
|
|
" before L.",
|
|
" between L and M.",
|
|
" after L."
|
|
],
|
|
"tags": [
|
|
"order"
|
|
],
|
|
"prediction": 2,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.804980278015137,
|
|
"mean_logprob": -2.9024901390075684,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.521125793457031,
|
|
"mean_logprob": -3.1737085978190103,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -10.34437084197998,
|
|
"mean_logprob": -2.068874168395996,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.500274658203125,
|
|
"mean_logprob": -3.1667582194010415,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8336159706115724,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028135656999438652
|
|
},
|
|
{
|
|
"id": "logical_reasoning-013",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "Some poets are teachers. Every teacher reads. Therefore, some poets",
|
|
"continuations": [
|
|
" read.",
|
|
" never read.",
|
|
" are not teachers.",
|
|
" are all teachers."
|
|
],
|
|
"tags": [
|
|
"syllogism"
|
|
],
|
|
"prediction": 2,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -5.698014736175537,
|
|
"mean_logprob": -2.8490073680877686,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.821164131164551,
|
|
"mean_logprob": -2.6070547103881836,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.846439361572266,
|
|
"mean_logprob": -1.7116098403930664,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.271710395812988,
|
|
"mean_logprob": -2.067927598953247,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.35631775856018066,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03454536499884853
|
|
},
|
|
{
|
|
"id": "logical_reasoning-014",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 1.35,
|
|
"item_elo": 950,
|
|
"context": "Exactly one card is marked. Card X is not marked and Card Y is marked. Card Z is therefore",
|
|
"continuations": [
|
|
" also marked.",
|
|
" not marked.",
|
|
" the same as Y.",
|
|
" impossible to classify."
|
|
],
|
|
"tags": [
|
|
"constraint"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.6638665199279785,
|
|
"mean_logprob": -1.8879555066426594,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -1.8135178089141846,
|
|
"mean_logprob": -0.6045059363047282,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -19.507362365722656,
|
|
"mean_logprob": -3.901472473144531,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -23.16461181640625,
|
|
"mean_logprob": -4.63292236328125,
|
|
"total_sequence_tokens": 26
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.2834495703379312,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027377290000003995
|
|
},
|
|
{
|
|
"id": "logical_reasoning-015",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 1.35,
|
|
"item_elo": 950,
|
|
"context": "If a number is divisible by four, it is even. Twenty is divisible by four, so twenty is",
|
|
"continuations": [
|
|
" odd.",
|
|
" prime.",
|
|
" even.",
|
|
" negative."
|
|
],
|
|
"tags": [
|
|
"deduction"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.139820098876953,
|
|
"mean_logprob": -3.0699100494384766,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.674606323242188,
|
|
"mean_logprob": -5.337303161621094,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -1.3192028999328613,
|
|
"mean_logprob": -0.6596014499664307,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -15.370566368103027,
|
|
"mean_logprob": -7.685283184051514,
|
|
"total_sequence_tokens": 24
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.410308599472046,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025267508999604615
|
|
},
|
|
{
|
|
"id": "logical_reasoning-016",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.35,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "No blue item is fragile. Vase V is fragile. Therefore, Vase V is",
|
|
"continuations": [
|
|
" definitely blue.",
|
|
" both blue and fragile.",
|
|
" not fragile.",
|
|
" not blue."
|
|
],
|
|
"tags": [
|
|
"contraposition"
|
|
],
|
|
"prediction": 2,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -13.683470726013184,
|
|
"mean_logprob": -4.5611569086710615,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -9.270719528198242,
|
|
"mean_logprob": -1.5451199213663738,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.348690032958984,
|
|
"mean_logprob": -1.087172508239746,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -7.262202262878418,
|
|
"mean_logprob": -2.420734087626139,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4579474131266277,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02956127200013725
|
|
},
|
|
{
|
|
"id": "logical_reasoning-017",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "All glims are nords. Some nords are pelts. From these facts alone, it follows that",
|
|
"continuations": [
|
|
" no conclusion links glims to pelts.",
|
|
" all glims are pelts.",
|
|
" some glims are pelts.",
|
|
" no nords are glims."
|
|
],
|
|
"tags": [
|
|
"quantifiers"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 9,
|
|
"sum_logprob": -37.376155853271484,
|
|
"mean_logprob": -4.152906205919054,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -13.048042297363281,
|
|
"mean_logprob": -1.8640060424804688,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -16.346662521362305,
|
|
"mean_logprob": -2.335237503051758,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.042264938354492,
|
|
"mean_logprob": -2.4346092769077847,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.47123146057128906,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0377511910010071
|
|
},
|
|
{
|
|
"id": "logical_reasoning-018",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "If R then S. If S then T. R is true. Therefore,",
|
|
"continuations": [
|
|
" T is false.",
|
|
" T is true.",
|
|
" S is false.",
|
|
" R is false."
|
|
],
|
|
"tags": [
|
|
"conditional_chain"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -3.9769163131713867,
|
|
"mean_logprob": -0.9942290782928467,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -3.0385985374450684,
|
|
"mean_logprob": -0.7596496343612671,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.500736236572266,
|
|
"mean_logprob": -1.3751840591430664,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.0516676902771,
|
|
"mean_logprob": -1.512916922569275,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.2345794439315796,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.035311272000399185
|
|
},
|
|
{
|
|
"id": "logical_reasoning-019",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "Exactly two of A, B, and C are true. A is true and B is false. Therefore, C is",
|
|
"continuations": [
|
|
" false.",
|
|
" unknown.",
|
|
" true.",
|
|
" both true and false."
|
|
],
|
|
"tags": [
|
|
"constraint"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -0.36741650104522705,
|
|
"mean_logprob": -0.18370825052261353,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -10.698878288269043,
|
|
"mean_logprob": -5.3494391441345215,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -1.841135859489441,
|
|
"mean_logprob": -0.9205679297447205,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -7.60679292678833,
|
|
"mean_logprob": -1.521358585357666,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7368596792221069,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03049728999940271
|
|
},
|
|
{
|
|
"id": "logical_reasoning-020",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "Kai sits somewhere left of Lio. Mina sits right of Lio. Which statement must be true?",
|
|
"continuations": [
|
|
" Mina sits left of Kai.",
|
|
" Kai sits beside Mina.",
|
|
" Lio sits left of Kai.",
|
|
" Kai sits left of Mina."
|
|
],
|
|
"tags": [
|
|
"spatial"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 9,
|
|
"sum_logprob": -20.981042861938477,
|
|
"mean_logprob": -2.3312269846598306,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 9,
|
|
"sum_logprob": -22.95328140258789,
|
|
"mean_logprob": -2.5503646002875433,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 9,
|
|
"sum_logprob": -21.372692108154297,
|
|
"mean_logprob": -2.3747435675726996,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 9,
|
|
"sum_logprob": -21.874128341674805,
|
|
"mean_logprob": -2.4304587046305337,
|
|
"total_sequence_tokens": 34
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.04351658291286897,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03377929599992058
|
|
},
|
|
{
|
|
"id": "logical_reasoning-021",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "Every zor is a vek, and no vek is a lum. Therefore,",
|
|
"continuations": [
|
|
" no zor is a lum.",
|
|
" every lum is a zor.",
|
|
" some zors are lums.",
|
|
" no zor is a vek."
|
|
],
|
|
"tags": [
|
|
"syllogism"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -9.927262306213379,
|
|
"mean_logprob": -1.2409077882766724,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -12.893324851989746,
|
|
"mean_logprob": -1.6116656064987183,
|
|
"total_sequence_tokens": 27
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -23.006277084350586,
|
|
"mean_logprob": -3.2866110120500838,
|
|
"total_sequence_tokens": 26
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -12.96645736694336,
|
|
"mean_logprob": -1.62080717086792,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3707578182220459,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.029061327000817982
|
|
},
|
|
{
|
|
"id": "logical_reasoning-022",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "The alarm rings if and only if the sensor is active. The alarm is silent. Thus the sensor is",
|
|
"continuations": [
|
|
" active.",
|
|
" inactive.",
|
|
" both states.",
|
|
" unrelated to the alarm."
|
|
],
|
|
"tags": [
|
|
"biconditional"
|
|
],
|
|
"prediction": 3,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.9675488471984863,
|
|
"mean_logprob": -1.9837744235992432,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.27786111831665,
|
|
"mean_logprob": -2.138930559158325,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -24.123218536376953,
|
|
"mean_logprob": -8.041072845458984,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -11.43774127960205,
|
|
"mean_logprob": -1.9062902132670085,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.07748421033223463,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026720802999989246
|
|
},
|
|
{
|
|
"id": "logical_reasoning-023",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "At least one of P or Q is false. P is true. It follows that Q is",
|
|
"continuations": [
|
|
" true.",
|
|
" undetermined.",
|
|
" false.",
|
|
" equal to P."
|
|
],
|
|
"tags": [
|
|
"propositional"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -1.5732948780059814,
|
|
"mean_logprob": -0.7866474390029907,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.106487274169922,
|
|
"mean_logprob": -4.2766218185424805,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -0.8197979927062988,
|
|
"mean_logprob": -0.4098989963531494,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.352224349975586,
|
|
"mean_logprob": -2.5880560874938965,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3767484426498413,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025991026999690803
|
|
},
|
|
{
|
|
"id": "logical_reasoning-024",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "Three runners finish with no ties. Ada is not first. Bo finishes after Cy. The only possible first-place runner is",
|
|
"continuations": [
|
|
" Ada.",
|
|
" Bo.",
|
|
" either Ada or Bo.",
|
|
" Cy."
|
|
],
|
|
"tags": [
|
|
"constraint"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.815156936645508,
|
|
"mean_logprob": -3.907578468322754,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -7.207143783569336,
|
|
"mean_logprob": -3.603571891784668,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -18.599592208862305,
|
|
"mean_logprob": -3.719918441772461,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.466699123382568,
|
|
"mean_logprob": -3.233349561691284,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3702223300933838,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026135096999496454
|
|
},
|
|
{
|
|
"id": "logical_reasoning-025",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "If the report is approved, funding continues. Funding did not continue. Therefore, the report was",
|
|
"continuations": [
|
|
" not approved.",
|
|
" approved late.",
|
|
" necessarily reviewed.",
|
|
" unrelated to funding."
|
|
],
|
|
"tags": [
|
|
"modus_tollens"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.09077262878418,
|
|
"mean_logprob": -1.6969242095947266,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -17.806264877319336,
|
|
"mean_logprob": -5.935421625773112,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -21.076892852783203,
|
|
"mean_logprob": -7.025630950927734,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -18.95391273498535,
|
|
"mean_logprob": -3.7907825469970704,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.093858337402344,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.06509407600060513
|
|
},
|
|
{
|
|
"id": "logical_reasoning-026",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "Some artists are engineers, and no engineer is careless. Therefore, some artists are",
|
|
"continuations": [
|
|
" careless.",
|
|
" not careless.",
|
|
" not engineers.",
|
|
" all engineers."
|
|
],
|
|
"tags": [
|
|
"syllogism"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.2993855476379395,
|
|
"mean_logprob": -2.09979518254598,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -7.499154090881348,
|
|
"mean_logprob": -1.874788522720337,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.7073564529418945,
|
|
"mean_logprob": -1.9024521509806316,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -9.565094947814941,
|
|
"mean_logprob": -3.1883649826049805,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.02766362826029467,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.06115498600047431
|
|
},
|
|
{
|
|
"id": "logical_reasoning-027",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "A statement and its negation cannot both be true. The statement is true, so its negation is",
|
|
"continuations": [
|
|
" true.",
|
|
" unknown.",
|
|
" false.",
|
|
" equivalent."
|
|
],
|
|
"tags": [
|
|
"negation"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.395155906677246,
|
|
"mean_logprob": -2.197577953338623,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -14.564260482788086,
|
|
"mean_logprob": -7.282130241394043,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -0.576337456703186,
|
|
"mean_logprob": -0.288168728351593,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -14.92807674407959,
|
|
"mean_logprob": -7.464038372039795,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.90940922498703,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0335494060000201
|
|
},
|
|
{
|
|
"id": "logical_reasoning-028",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "If X is selected, Y must be selected. If Y is selected, Z cannot be selected. X is selected, so Z",
|
|
"continuations": [
|
|
" must be selected.",
|
|
" may equal X.",
|
|
" makes Y unselected.",
|
|
" cannot be selected."
|
|
],
|
|
"tags": [
|
|
"constraint"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -1.6457048654556274,
|
|
"mean_logprob": -0.41142621636390686,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.842538833618164,
|
|
"mean_logprob": -4.460634708404541,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -15.62555980682373,
|
|
"mean_logprob": -3.125111961364746,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -4.505889415740967,
|
|
"mean_logprob": -1.1264723539352417,
|
|
"total_sequence_tokens": 30
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7150461375713348,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03742306400090456
|
|
},
|
|
{
|
|
"id": "logical_reasoning-029",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "No historians are timeless beings. Some writers are historians. Therefore,",
|
|
"continuations": [
|
|
" some writers are not timeless beings.",
|
|
" all writers are timeless.",
|
|
" no writers are historians.",
|
|
" all historians are writers."
|
|
],
|
|
"tags": [
|
|
"syllogism"
|
|
],
|
|
"prediction": 3,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -24.91695785522461,
|
|
"mean_logprob": -3.114619731903076,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -15.53573989868164,
|
|
"mean_logprob": -2.589289983113607,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -12.175522804260254,
|
|
"mean_logprob": -2.435104560852051,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -11.63157844543457,
|
|
"mean_logprob": -2.326315689086914,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.10878887176513707,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028489014999649953
|
|
},
|
|
{
|
|
"id": "logical_reasoning-030",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "Either J or K, but not both, is open. If J is open, L is closed. J is open. Hence",
|
|
"continuations": [
|
|
" K and L are open.",
|
|
" K and L are closed.",
|
|
" K is open and L closed.",
|
|
" K is closed and L open."
|
|
],
|
|
"tags": [
|
|
"propositional"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -11.102985382080078,
|
|
"mean_logprob": -1.850497563680013,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -10.669707298278809,
|
|
"mean_logprob": -1.7782845497131348,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -13.543540954589844,
|
|
"mean_logprob": -1.9347915649414062,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -15.956045150756836,
|
|
"mean_logprob": -2.279435021536691,
|
|
"total_sequence_tokens": 34
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.07221301396687818,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.039278928999920026
|
|
},
|
|
{
|
|
"id": "logical_reasoning-031",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "Rhea is before Sami. Taro is after Sami. Ula is before Rhea. The second person in order is",
|
|
"continuations": [
|
|
" Ula.",
|
|
" Sami.",
|
|
" Rhea.",
|
|
" Taro."
|
|
],
|
|
"tags": [
|
|
"order"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -3.3497650623321533,
|
|
"mean_logprob": -1.1165883541107178,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.953437566757202,
|
|
"mean_logprob": -0.9844791889190674,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.262697458267212,
|
|
"mean_logprob": -0.7542324860890707,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.8338801860809326,
|
|
"mean_logprob": -0.9446267286936442,
|
|
"total_sequence_tokens": 30
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.1903942426045735,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02460370500011777
|
|
},
|
|
{
|
|
"id": "logical_reasoning-032",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "All valid tickets have a stamp. This ticket has no stamp. It is therefore",
|
|
"continuations": [
|
|
" valid.",
|
|
" stamped invisibly.",
|
|
" both valid and invalid.",
|
|
" not valid."
|
|
],
|
|
"tags": [
|
|
"contraposition"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -2.1518657207489014,
|
|
"mean_logprob": -1.0759328603744507,
|
|
"total_sequence_tokens": 18
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -25.669761657714844,
|
|
"mean_logprob": -4.278293609619141,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -8.00202751159668,
|
|
"mean_logprob": -1.600405502319336,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.469339370727539,
|
|
"mean_logprob": -0.8231131235758463,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.25281973679860437,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02711293900028977
|
|
},
|
|
{
|
|
"id": "logical_reasoning-033",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0250000000000004,
|
|
"earned_points": 2.0250000000000004,
|
|
"item_elo": 1050,
|
|
"context": "Exactly one of the claims \"A is true\" and \"B is false\" is true. A is false. Therefore, B is",
|
|
"continuations": [
|
|
" false.",
|
|
" true.",
|
|
" both.",
|
|
" undetermined."
|
|
],
|
|
"tags": [
|
|
"propositional"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -0.25394731760025024,
|
|
"mean_logprob": -0.12697365880012512,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.023630380630493,
|
|
"mean_logprob": -1.5118151903152466,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 2,
|
|
"sum_logprob": -17.4687557220459,
|
|
"mean_logprob": -8.73437786102295,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.809072494506836,
|
|
"mean_logprob": -4.452268123626709,
|
|
"total_sequence_tokens": 33
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.3848415315151215,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03756342099950416
|
|
},
|
|
{
|
|
"id": "logical_reasoning-034",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "Every quill is a renn. No renn is a soth. Some varks are quills. It follows that some varks are",
|
|
"continuations": [
|
|
" soths.",
|
|
" not soths.",
|
|
" not quills.",
|
|
" all renns."
|
|
],
|
|
"tags": [
|
|
"syllogism"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 3,
|
|
"sum_logprob": -5.736642837524414,
|
|
"mean_logprob": -1.9122142791748047,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.529621124267578,
|
|
"mean_logprob": -2.1324052810668945,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -3.5806849002838135,
|
|
"mean_logprob": -0.8951712250709534,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -20.263328552246094,
|
|
"mean_logprob": -5.065832138061523,
|
|
"total_sequence_tokens": 34
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.0170430541038513,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.037289870999302366
|
|
},
|
|
{
|
|
"id": "logical_reasoning-035",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "Exactly two of P, Q, R, and S are true. P and Q have opposite truth values. R is true and S is false. Therefore,",
|
|
"continuations": [
|
|
" P and Q are both true.",
|
|
" P and Q are both false.",
|
|
" exactly one of P and Q is true.",
|
|
" neither P nor Q matters."
|
|
],
|
|
"tags": [
|
|
"constraint"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -6.264081954956055,
|
|
"mean_logprob": -0.8948688507080078,
|
|
"total_sequence_tokens": 39
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -6.364994525909424,
|
|
"mean_logprob": -0.9092849322727748,
|
|
"total_sequence_tokens": 39
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 9,
|
|
"sum_logprob": -19.117538452148438,
|
|
"mean_logprob": -2.124170939127604,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -13.39778995513916,
|
|
"mean_logprob": -2.2329649925231934,
|
|
"total_sequence_tokens": 38
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.014416081564767036,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03755214200100454
|
|
},
|
|
{
|
|
"id": "logical_reasoning-036",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "If A then B. If not C then not B. A is true. Which conclusion must follow?",
|
|
"continuations": [
|
|
" C is false.",
|
|
" B is false.",
|
|
" A is false.",
|
|
" C is true."
|
|
],
|
|
"tags": [
|
|
"conditional_chain"
|
|
],
|
|
"prediction": 2,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.453317642211914,
|
|
"mean_logprob": -4.3633294105529785,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.41801643371582,
|
|
"mean_logprob": -4.354504108428955,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.327856063842773,
|
|
"mean_logprob": -4.081964015960693,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -17.262537002563477,
|
|
"mean_logprob": -4.315634250640869,
|
|
"total_sequence_tokens": 25
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.23367023468017578,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027254909999101073
|
|
},
|
|
{
|
|
"id": "logical_reasoning-037",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 3.0375,
|
|
"item_elo": 1200,
|
|
"context": "Five seats are numbered 1 through 5. Iva sits immediately left of Jae. Kian sits in seat 5. Iva sits in seat 2. Jae sits in",
|
|
"continuations": [
|
|
" seat 3.",
|
|
" seat 1.",
|
|
" seat 4.",
|
|
" seat 5."
|
|
],
|
|
"tags": [
|
|
"constraint"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -1.0388476848602295,
|
|
"mean_logprob": -0.2597119212150574,
|
|
"total_sequence_tokens": 48
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -1.3396497964859009,
|
|
"mean_logprob": -0.3349124491214752,
|
|
"total_sequence_tokens": 48
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -1.67790687084198,
|
|
"mean_logprob": -0.419476717710495,
|
|
"total_sequence_tokens": 48
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.073331356048584,
|
|
"mean_logprob": -1.268332839012146,
|
|
"total_sequence_tokens": 48
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.07520052790641785,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03574124800070422
|
|
},
|
|
{
|
|
"id": "logical_reasoning-038",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "No fal is a mer. Every tov is a fal. Some riks are mers. Which statement must be true?",
|
|
"continuations": [
|
|
" Some riks are tovs.",
|
|
" No tov is a mer.",
|
|
" Every rik is a fal.",
|
|
" Some mers are fals."
|
|
],
|
|
"tags": [
|
|
"quantifiers"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -20.39111328125,
|
|
"mean_logprob": -2.9130161830357144,
|
|
"total_sequence_tokens": 35
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -18.03400230407715,
|
|
"mean_logprob": -2.5762860434395924,
|
|
"total_sequence_tokens": 35
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 8,
|
|
"sum_logprob": -15.599857330322266,
|
|
"mean_logprob": -1.9499821662902832,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.913246154785156,
|
|
"mean_logprob": -2.559035164969308,
|
|
"total_sequence_tokens": 35
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.609052998679025,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0381310630000371
|
|
},
|
|
{
|
|
"id": "logical_reasoning-039",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "A safe opens exactly when both keys are turned. The safe did not open, and the first key was turned. Therefore, the second key was",
|
|
"continuations": [
|
|
" turned.",
|
|
" broken necessarily.",
|
|
" not turned.",
|
|
" unrelated."
|
|
],
|
|
"tags": [
|
|
"conjunction"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -3.114652633666992,
|
|
"mean_logprob": -1.557326316833496,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -24.938669204711914,
|
|
"mean_logprob": -8.31288973490397,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -6.651601314544678,
|
|
"mean_logprob": -2.217200438181559,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 3,
|
|
"sum_logprob": -13.41206169128418,
|
|
"mean_logprob": -4.47068723042806,
|
|
"total_sequence_tokens": 32
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.659874121348063,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.025557093000315945
|
|
},
|
|
{
|
|
"id": "logical_reasoning-040",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 3.0375,
|
|
"item_elo": 1200,
|
|
"context": "A committee selects exactly three of J, K, L, and M. If J is selected, K is not. L and M are selected. Therefore, the third member must be",
|
|
"continuations": [
|
|
" J.",
|
|
" L again.",
|
|
" M again.",
|
|
" K."
|
|
],
|
|
"tags": [
|
|
"constraint"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -6.013801574707031,
|
|
"mean_logprob": -3.0069007873535156,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -14.643722534179688,
|
|
"mean_logprob": -4.8812408447265625,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 3,
|
|
"sum_logprob": -15.19642448425293,
|
|
"mean_logprob": -5.06547482808431,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.991424560546875,
|
|
"mean_logprob": -2.4957122802734375,
|
|
"total_sequence_tokens": 40
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5111885070800781,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03555398400021659
|
|
},
|
|
{
|
|
"id": "logical_reasoning-041",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "Some nels are paks. All paks are wims. No wim is a dor. Therefore,",
|
|
"continuations": [
|
|
" some nels are not dors.",
|
|
" all nels are wims.",
|
|
" some dors are paks.",
|
|
" no nels are paks."
|
|
],
|
|
"tags": [
|
|
"syllogism"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -17.275667190551758,
|
|
"mean_logprob": -2.1594583988189697,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -7.622884750366211,
|
|
"mean_logprob": -1.0889835357666016,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -15.269927978515625,
|
|
"mean_logprob": -2.1814182826450894,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -11.179550170898438,
|
|
"mean_logprob": -1.597078595842634,
|
|
"total_sequence_tokens": 31
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5080950600760323,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03802208500019333
|
|
},
|
|
{
|
|
"id": "logical_reasoning-042",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "Four tasks W, X, Y, Z are scheduled once each. W precedes X. Y immediately precedes Z. X is first. These conditions are",
|
|
"continuations": [
|
|
" satisfied by X, W, Y, Z.",
|
|
" impossible to satisfy.",
|
|
" satisfied by W, X, Y, Z.",
|
|
" satisfied by Y, Z, W, X."
|
|
],
|
|
"tags": [
|
|
"consistency"
|
|
],
|
|
"prediction": 2,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 10,
|
|
"sum_logprob": -18.201345443725586,
|
|
"mean_logprob": -1.8201345443725585,
|
|
"total_sequence_tokens": 42
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -10.938289642333984,
|
|
"mean_logprob": -2.734572410583496,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 10,
|
|
"sum_logprob": -15.769869804382324,
|
|
"mean_logprob": -1.5769869804382324,
|
|
"total_sequence_tokens": 42
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 10,
|
|
"sum_logprob": -23.666305541992188,
|
|
"mean_logprob": -2.3666305541992188,
|
|
"total_sequence_tokens": 42
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.24314756393432613,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04649977599910926
|
|
},
|
|
{
|
|
"id": "logical_reasoning-043",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 3.0375,
|
|
"item_elo": 1200,
|
|
"context": "If either M or N is true, P is true. P is false. Therefore,",
|
|
"continuations": [
|
|
" M is true.",
|
|
" N is true.",
|
|
" both M and N are false.",
|
|
" exactly one is false."
|
|
],
|
|
"tags": [
|
|
"contraposition"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.543510437011719,
|
|
"mean_logprob": -2.3858776092529297,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.155370712280273,
|
|
"mean_logprob": -2.2888426780700684,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -6.482165336608887,
|
|
"mean_logprob": -0.9260236195155552,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -15.393881797790527,
|
|
"mean_logprob": -3.0787763595581055,
|
|
"total_sequence_tokens": 23
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.3628190585545132,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026972736000971054
|
|
},
|
|
{
|
|
"id": "logical_reasoning-044",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "All A-things are B-things. Some B-things are C-things. No C-thing is a D-thing. Which claim is guaranteed?",
|
|
"continuations": [
|
|
" Some A-things are C-things.",
|
|
" No A-thing is a D-thing.",
|
|
" All B-things are A-things.",
|
|
" Some B-things are not D-things."
|
|
],
|
|
"tags": [
|
|
"quantifiers"
|
|
],
|
|
"prediction": 2,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 11,
|
|
"sum_logprob": -12.665159225463867,
|
|
"mean_logprob": -1.1513781114058061,
|
|
"total_sequence_tokens": 48
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 10,
|
|
"sum_logprob": -14.303797721862793,
|
|
"mean_logprob": -1.4303797721862792,
|
|
"total_sequence_tokens": 47
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 11,
|
|
"sum_logprob": -10.99329662322998,
|
|
"mean_logprob": -0.9993906021118164,
|
|
"total_sequence_tokens": 48
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 12,
|
|
"sum_logprob": -14.286218643188477,
|
|
"mean_logprob": -1.1905182202657063,
|
|
"total_sequence_tokens": 49
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.15198750929398974,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04841408700121974
|
|
},
|
|
{
|
|
"id": "logical_reasoning-045",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 3.0375,
|
|
"item_elo": 1200,
|
|
"context": "A code is valid if exactly one of conditions X and Y holds. X holds and the code is valid. Therefore, Y",
|
|
"continuations": [
|
|
" does not hold.",
|
|
" also holds.",
|
|
" is equivalent to X.",
|
|
" cannot be discussed."
|
|
],
|
|
"tags": [
|
|
"exclusive_or"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -3.2396998405456543,
|
|
"mean_logprob": -0.8099249601364136,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -4.657301425933838,
|
|
"mean_logprob": -1.5524338086446126,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -10.75133991241455,
|
|
"mean_logprob": -2.15026798248291,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -20.620361328125,
|
|
"mean_logprob": -5.15509033203125,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.742508848508199,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.031652417999794125
|
|
},
|
|
{
|
|
"id": "logical_reasoning-046",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 3.0375,
|
|
"item_elo": 1200,
|
|
"context": "R is earlier than S. T is later than U. S is earlier than U. Which ordering must hold?",
|
|
"continuations": [
|
|
" T is earlier than R.",
|
|
" R is earlier than T.",
|
|
" U is earlier than S.",
|
|
" S is later than T."
|
|
],
|
|
"tags": [
|
|
"transitive"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -20.732337951660156,
|
|
"mean_logprob": -3.455389658610026,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -14.730389595031738,
|
|
"mean_logprob": -2.4550649325052896,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -20.681095123291016,
|
|
"mean_logprob": -3.4468491872151694,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -16.98881721496582,
|
|
"mean_logprob": -2.8314695358276367,
|
|
"total_sequence_tokens": 29
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.37640460332234715,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05448604800039902
|
|
},
|
|
{
|
|
"id": "logical_reasoning-047",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "If the engine starts, either fuel or battery power is available. The engine starts, and no fuel is available. Thus",
|
|
"continuations": [
|
|
" no power is available.",
|
|
" the engine did not start.",
|
|
" battery power is available.",
|
|
" fuel is available."
|
|
],
|
|
"tags": [
|
|
"disjunction"
|
|
],
|
|
"prediction": 0,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 5,
|
|
"sum_logprob": -21.319812774658203,
|
|
"mean_logprob": -4.2639625549316404,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -25.933473587036133,
|
|
"mean_logprob": -4.3222455978393555,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -23.158662796020508,
|
|
"mean_logprob": -4.631732559204101,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -22.04351806640625,
|
|
"mean_logprob": -5.5108795166015625,
|
|
"total_sequence_tokens": 28
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.05828304290771502,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03828422100013995
|
|
},
|
|
{
|
|
"id": "logical_reasoning-048",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "Exactly one person among A, B, and C lies. A says B tells the truth. B says C tells the truth. C says A lies. The liar is",
|
|
"continuations": [
|
|
" A.",
|
|
" B.",
|
|
" impossible to determine.",
|
|
" C."
|
|
],
|
|
"tags": [
|
|
"truth_tellers"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -4.278550624847412,
|
|
"mean_logprob": -2.139275312423706,
|
|
"total_sequence_tokens": 37
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -8.535700798034668,
|
|
"mean_logprob": -4.267850399017334,
|
|
"total_sequence_tokens": 37
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -15.505983352661133,
|
|
"mean_logprob": -3.876495838165283,
|
|
"total_sequence_tokens": 39
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 2,
|
|
"sum_logprob": -11.253164291381836,
|
|
"mean_logprob": -5.626582145690918,
|
|
"total_sequence_tokens": 37
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.7372205257415771,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03660139399835316
|
|
},
|
|
{
|
|
"id": "logical_reasoning-049",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "No glens are haps. Some jors are glens. Every jor is a mip. It follows that some mips are",
|
|
"continuations": [
|
|
" not haps.",
|
|
" haps.",
|
|
" not jors.",
|
|
" all glens."
|
|
],
|
|
"tags": [
|
|
"syllogism"
|
|
],
|
|
"prediction": 1,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.938095092773438,
|
|
"mean_logprob": -2.2345237731933594,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 3,
|
|
"sum_logprob": -2.8486106395721436,
|
|
"mean_logprob": -0.9495368798573812,
|
|
"total_sequence_tokens": 32
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -8.137388229370117,
|
|
"mean_logprob": -2.0343470573425293,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -5.661764144897461,
|
|
"mean_logprob": -1.4154410362243652,
|
|
"total_sequence_tokens": 33
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.465904156366984,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03652822299955005
|
|
},
|
|
{
|
|
"id": "logical_reasoning-050",
|
|
"category": "logical_reasoning",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.35,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.0375,
|
|
"earned_points": 3.0375,
|
|
"item_elo": 1200,
|
|
"context": "A proposition is true only if Q is true, and Q is true only if R is false. The proposition is true. Therefore, R is",
|
|
"continuations": [
|
|
" true.",
|
|
" false.",
|
|
" both true and false.",
|
|
" independent of Q."
|
|
],
|
|
"tags": [
|
|
"conditional_chain"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 2,
|
|
"sum_logprob": -2.581470012664795,
|
|
"mean_logprob": -1.2907350063323975,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 2,
|
|
"sum_logprob": -0.4033554494380951,
|
|
"mean_logprob": -0.20167772471904755,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 5,
|
|
"sum_logprob": -5.504075527191162,
|
|
"mean_logprob": -1.1008151054382325,
|
|
"total_sequence_tokens": 34
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.2425537109375,
|
|
"mean_logprob": -2.310638427734375,
|
|
"total_sequence_tokens": 33
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8991373807191849,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03524178599946026
|
|
},
|
|
{
|
|
"id": "code_completion-001",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def square(x):\n",
|
|
"continuations": [
|
|
" return x * x\n",
|
|
" return x + x\n",
|
|
" print(x)\n",
|
|
" return 2\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -5.934330463409424,
|
|
"mean_logprob": -0.8477614947727748,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -14.469461441040039,
|
|
"mean_logprob": -2.067065920148577,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -9.781411170959473,
|
|
"mean_logprob": -1.630235195159912,
|
|
"total_sequence_tokens": 12
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 5,
|
|
"sum_logprob": -18.043697357177734,
|
|
"mean_logprob": -3.608739471435547,
|
|
"total_sequence_tokens": 11
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7824737003871373,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026375286000984488
|
|
},
|
|
{
|
|
"id": "code_completion-002",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def is_empty(items):\n",
|
|
"continuations": [
|
|
" return items[0]\n",
|
|
" return len(items) == 0\n",
|
|
" return len(items) + 1\n",
|
|
" items.clear()\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -14.702980995178223,
|
|
"mean_logprob": -2.1004258564540317,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 11,
|
|
"sum_logprob": -8.024712562561035,
|
|
"mean_logprob": -0.729519323869185,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 11,
|
|
"sum_logprob": -16.210472106933594,
|
|
"mean_logprob": -1.4736792824485085,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -9.871957778930664,
|
|
"mean_logprob": -1.645326296488444,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7441599585793235,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0282552590015257
|
|
},
|
|
{
|
|
"id": "code_completion-003",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def greet(name):\n",
|
|
"continuations": [
|
|
" return name - \"Hello\"\n",
|
|
" print = name\n",
|
|
" return \"Hello, \" + name\n",
|
|
" return len(\"Hello\")\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 10,
|
|
"sum_logprob": -24.306949615478516,
|
|
"mean_logprob": -2.4306949615478515,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -19.574186325073242,
|
|
"mean_logprob": -3.262364387512207,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 12,
|
|
"sum_logprob": -11.44089412689209,
|
|
"mean_logprob": -0.9534078439076742,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -27.81157684326172,
|
|
"mean_logprob": -3.9730824061802457,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.4772871176401772,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03970513099920936
|
|
},
|
|
{
|
|
"id": "code_completion-004",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def first(items):\n",
|
|
"continuations": [
|
|
" return items[-1]\n",
|
|
" return len(items)\n",
|
|
" return items\n",
|
|
" return items[0]\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -15.152753829956055,
|
|
"mean_logprob": -2.1646791185651506,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -14.308820724487305,
|
|
"mean_logprob": -2.0441172463553294,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 4,
|
|
"sum_logprob": -9.552918434143066,
|
|
"mean_logprob": -2.3882296085357666,
|
|
"total_sequence_tokens": 10
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -11.064523696899414,
|
|
"mean_logprob": -1.580646242414202,
|
|
"total_sequence_tokens": 13
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4634710039411274,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027532070000233944
|
|
},
|
|
{
|
|
"id": "code_completion-005",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def add(a, b):\n",
|
|
"continuations": [
|
|
" return a + b\n",
|
|
" return a - b\n",
|
|
" return a == b\n",
|
|
" return [a, b]\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -8.011276245117188,
|
|
"mean_logprob": -1.144468035016741,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.55638313293457,
|
|
"mean_logprob": -2.508054733276367,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -19.319067001342773,
|
|
"mean_logprob": -2.759866714477539,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 9,
|
|
"sum_logprob": -14.94140625,
|
|
"mean_logprob": -1.66015625,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5156882149832589,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026762985999084776
|
|
},
|
|
{
|
|
"id": "code_completion-006",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def last(items):\n",
|
|
"continuations": [
|
|
" return items[0]\n",
|
|
" return items[-1]\n",
|
|
" return items[1:]\n",
|
|
" return -len(items)\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -12.57072639465332,
|
|
"mean_logprob": -1.7958180563790458,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -10.278488159179688,
|
|
"mean_logprob": -1.468355451311384,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -13.712234497070312,
|
|
"mean_logprob": -1.958890642438616,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 9,
|
|
"sum_logprob": -18.184860229492188,
|
|
"mean_logprob": -2.020540025499132,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.3274626050676619,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03602598600082274
|
|
},
|
|
{
|
|
"id": "code_completion-007",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def is_even(n):\n",
|
|
"continuations": [
|
|
" return n / 2\n",
|
|
" return n % 2 == 1\n",
|
|
" return n % 2 == 0\n",
|
|
" return n * 2\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -24.60840606689453,
|
|
"mean_logprob": -3.0760507583618164,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 12,
|
|
"sum_logprob": -19.844167709350586,
|
|
"mean_logprob": -1.653680642445882,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 12,
|
|
"sum_logprob": -6.369754791259766,
|
|
"mean_logprob": -0.5308128992716471,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -24.57311248779297,
|
|
"mean_logprob": -3.071639060974121,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.1228677431742349,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02552455099976214
|
|
},
|
|
{
|
|
"id": "code_completion-008",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def absolute_value(x):\n",
|
|
"continuations": [
|
|
" return -abs(x)\n",
|
|
" return x * 0\n",
|
|
" return bool(x)\n",
|
|
" return abs(x)\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 9,
|
|
"sum_logprob": -11.221795082092285,
|
|
"mean_logprob": -1.2468661202324762,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -19.985536575317383,
|
|
"mean_logprob": -2.498192071914673,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -15.934894561767578,
|
|
"mean_logprob": -2.2764135088239397,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -6.373825550079346,
|
|
"mean_logprob": -0.9105465071541923,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.33631961307828395,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.027187195999431424
|
|
},
|
|
{
|
|
"id": "code_completion-009",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def double_all(numbers):\n",
|
|
"continuations": [
|
|
" return [n * 2 for n in numbers]\n",
|
|
" return numbers * 0\n",
|
|
" return [2 for n in numbers]\n",
|
|
" return len(numbers) * 2\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 15,
|
|
"sum_logprob": -13.590858459472656,
|
|
"mean_logprob": -0.9060572306315104,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -21.125938415527344,
|
|
"mean_logprob": -2.640742301940918,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 11,
|
|
"sum_logprob": -25.994300842285156,
|
|
"mean_logprob": -2.3631182583895596,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 11,
|
|
"sum_logprob": -19.167556762695312,
|
|
"mean_logprob": -1.7425051602450283,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8364479296135179,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04008716399948753
|
|
},
|
|
{
|
|
"id": "code_completion-010",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def contains(items, value):\n",
|
|
"continuations": [
|
|
" return items == value\n",
|
|
" return value in items\n",
|
|
" return items + value\n",
|
|
" return items.index\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -10.957561492919922,
|
|
"mean_logprob": -1.565365927559989,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -8.056070327758789,
|
|
"mean_logprob": -1.342678387959798,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -13.39298152923584,
|
|
"mean_logprob": -1.91328307560512,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -27.022537231445312,
|
|
"mean_logprob": -4.503756205240886,
|
|
"total_sequence_tokens": 14
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.22268753960019083,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02570848799950909
|
|
},
|
|
{
|
|
"id": "code_completion-011",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def maximum(a, b):\n",
|
|
"continuations": [
|
|
" return a + b\n",
|
|
" return a < b\n",
|
|
" return a if a > b else b\n",
|
|
" return None\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -12.686381340026855,
|
|
"mean_logprob": -1.812340191432408,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -12.447408676147461,
|
|
"mean_logprob": -1.7782012394496374,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 11,
|
|
"sum_logprob": -11.49631404876709,
|
|
"mean_logprob": -1.0451194589788264,
|
|
"total_sequence_tokens": 19
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 4,
|
|
"sum_logprob": -16.26229476928711,
|
|
"mean_logprob": -4.065573692321777,
|
|
"total_sequence_tokens": 12
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.733081780470811,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02868215599846735
|
|
},
|
|
{
|
|
"id": "code_completion-012",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def repeat(text, count):\n",
|
|
"continuations": [
|
|
" return count / text\n",
|
|
" return len(text)\n",
|
|
" return text + count\n",
|
|
" return text * count\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -24.464187622070312,
|
|
"mean_logprob": -3.4948839460100447,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -15.841794967651367,
|
|
"mean_logprob": -2.263113566807338,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -15.606399536132812,
|
|
"mean_logprob": -2.229485648018973,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -11.067680358886719,
|
|
"mean_logprob": -1.581097194126674,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.648388453892299,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.026035105998744257
|
|
},
|
|
{
|
|
"id": "code_completion-013",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def keys_list(mapping):\n",
|
|
"continuations": [
|
|
" return list(mapping.keys())\n",
|
|
" return mapping.values\n",
|
|
" return [mapping]\n",
|
|
" return mapping.clear()\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 9,
|
|
"sum_logprob": -9.412752151489258,
|
|
"mean_logprob": -1.045861350165473,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -20.372913360595703,
|
|
"mean_logprob": -3.3954855600992837,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.14534568786621,
|
|
"mean_logprob": -2.4493350982666016,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -20.802125930786133,
|
|
"mean_logprob": -2.9717322758265903,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.4034737481011286,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.028490945000157808
|
|
},
|
|
{
|
|
"id": "code_completion-014",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def starts_with_a(text):\n",
|
|
"continuations": [
|
|
" return text[-1] == \"a\"\n",
|
|
" return text.startswith(\"a\")\n",
|
|
" return \"a\" + text\n",
|
|
" return text.count\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 13,
|
|
"sum_logprob": -20.738313674926758,
|
|
"mean_logprob": -1.595254898071289,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 11,
|
|
"sum_logprob": -11.440147399902344,
|
|
"mean_logprob": -1.040013399991122,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 10,
|
|
"sum_logprob": -15.75004768371582,
|
|
"mean_logprob": -1.575004768371582,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 6,
|
|
"sum_logprob": -20.53750991821289,
|
|
"mean_logprob": -3.4229183197021484,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5349913683804599,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03216659999998228
|
|
},
|
|
{
|
|
"id": "code_completion-015",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def total(numbers):\n",
|
|
"continuations": [
|
|
" return len(numbers)\n",
|
|
" return max(numbers)\n",
|
|
" return sum(numbers)\n",
|
|
" return numbers[0]\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -10.798761367797852,
|
|
"mean_logprob": -1.5426801953996931,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -11.562196731567383,
|
|
"mean_logprob": -1.651742390223912,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -6.728759288787842,
|
|
"mean_logprob": -0.9612513269696917,
|
|
"total_sequence_tokens": 13
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.91793441772461,
|
|
"mean_logprob": -2.559704916817801,
|
|
"total_sequence_tokens": 13
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5814288684300014,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02752010599942878
|
|
},
|
|
{
|
|
"id": "code_completion-016",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 0.0,
|
|
"item_elo": 950,
|
|
"context": "def clamp_zero(x):\n",
|
|
"continuations": [
|
|
" return min(0, x)\n",
|
|
" return x == 0\n",
|
|
" return -x\n",
|
|
" return max(0, x)\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 1,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 9,
|
|
"sum_logprob": -17.8612060546875,
|
|
"mean_logprob": -1.9845784505208333,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -8.925451278686523,
|
|
"mean_logprob": -1.1156814098358154,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -12.237079620361328,
|
|
"mean_logprob": -2.039513270060221,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 9,
|
|
"sum_logprob": -11.028650283813477,
|
|
"mean_logprob": -1.2254055870903864,
|
|
"total_sequence_tokens": 17
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.10972417725457095,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.02768399599881377
|
|
},
|
|
{
|
|
"id": "code_completion-017",
|
|
"category": "code_completion",
|
|
"difficulty": "easy",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.0,
|
|
"point_weight": 1.4,
|
|
"earned_points": 1.4,
|
|
"item_elo": 950,
|
|
"context": "def reversed_copy(items):\n",
|
|
"continuations": [
|
|
" return items[::-1]\n",
|
|
" return items[::1]\n",
|
|
" return items.reverse\n",
|
|
" return sorted(items)\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -7.552888870239258,
|
|
"mean_logprob": -0.9441111087799072,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -21.040647506713867,
|
|
"mean_logprob": -2.6300809383392334,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 6,
|
|
"sum_logprob": -18.220312118530273,
|
|
"mean_logprob": -3.0367186864217124,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -12.42491626739502,
|
|
"mean_logprob": -1.7749880381992884,
|
|
"total_sequence_tokens": 15
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.8308769294193812,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03015708699967945
|
|
},
|
|
{
|
|
"id": "code_completion-018",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def count_positive(numbers):\n",
|
|
"continuations": [
|
|
" return sum(numbers)\n",
|
|
" return sum(1 for n in numbers if n > 0)\n",
|
|
" return [n > 0 for n in numbers]\n",
|
|
" return len(numbers) > 0\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -12.093025207519531,
|
|
"mean_logprob": -1.7275750296456474,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 17,
|
|
"sum_logprob": -15.259834289550781,
|
|
"mean_logprob": -0.897637311150046,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 15,
|
|
"sum_logprob": -15.651079177856445,
|
|
"mean_logprob": -1.043405278523763,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 11,
|
|
"sum_logprob": -10.985816955566406,
|
|
"mean_logprob": -0.9987106323242188,
|
|
"total_sequence_tokens": 19
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.10107332117417278,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03531654699872888
|
|
},
|
|
{
|
|
"id": "code_completion-019",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def unique_in_order(items):\n seen = set()\n",
|
|
"continuations": [
|
|
" return set(items)\n",
|
|
" return sorted(items)\n",
|
|
" return [x for x in items if not (x in seen or seen.add(x))]\n",
|
|
" return list(reversed(items))\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -25.16358184814453,
|
|
"mean_logprob": -3.5947974068777904,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -16.42459487915039,
|
|
"mean_logprob": -2.3463706970214844,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 24,
|
|
"sum_logprob": -38.50417709350586,
|
|
"mean_logprob": -1.6043407122294109,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 10,
|
|
"sum_logprob": -23.19318389892578,
|
|
"mean_logprob": -2.319318389892578,
|
|
"total_sequence_tokens": 27
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.7149776776631673,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03663187399979506
|
|
},
|
|
{
|
|
"id": "code_completion-020",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "def safe_divide(a, b):\n if b == 0:\n",
|
|
"continuations": [
|
|
" return a\n return b / a\n",
|
|
" b = 1\n return b\n",
|
|
" raise StopIteration\n return a + b\n",
|
|
" return None\n return a / b\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 0,
|
|
"label": 3,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 11,
|
|
"sum_logprob": -14.361824989318848,
|
|
"mean_logprob": -1.3056204535744407,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 11,
|
|
"sum_logprob": -31.49257469177246,
|
|
"mean_logprob": -2.8629613356156782,
|
|
"total_sequence_tokens": 30
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 12,
|
|
"sum_logprob": -27.718412399291992,
|
|
"mean_logprob": -2.3098676999409995,
|
|
"total_sequence_tokens": 31
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 11,
|
|
"sum_logprob": -19.33892250061035,
|
|
"mean_logprob": -1.75808386369185,
|
|
"total_sequence_tokens": 30
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.4524634101174094,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04077255999982299
|
|
},
|
|
{
|
|
"id": "code_completion-021",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def word_lengths(words):\n",
|
|
"continuations": [
|
|
" return {word: len(word) for word in words}\n",
|
|
" return {len(words): word for word in words}\n",
|
|
" return [words, len(words)]\n",
|
|
" return words.keys()\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 16,
|
|
"sum_logprob": -14.82179069519043,
|
|
"mean_logprob": -0.9263619184494019,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 15,
|
|
"sum_logprob": -34.53876495361328,
|
|
"mean_logprob": -2.3025843302408853,
|
|
"total_sequence_tokens": 24
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 11,
|
|
"sum_logprob": -27.680112838745117,
|
|
"mean_logprob": -2.5163738944313745,
|
|
"total_sequence_tokens": 20
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -24.124025344848633,
|
|
"mean_logprob": -3.4462893349783763,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.3762224117914834,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03476182600024913
|
|
},
|
|
{
|
|
"id": "code_completion-022",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "def flatten_once(groups):\n",
|
|
"continuations": [
|
|
" return list(groups)\n",
|
|
" return [item for group in groups for item in group]\n",
|
|
" return [groups[0]]\n",
|
|
" return sum(len(g) for g in groups)\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 3,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -14.239151954650879,
|
|
"mean_logprob": -2.0341645649501254,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 15,
|
|
"sum_logprob": -24.19076156616211,
|
|
"mean_logprob": -1.6127174377441407,
|
|
"total_sequence_tokens": 23
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 9,
|
|
"sum_logprob": -18.42530059814453,
|
|
"mean_logprob": -2.047255622016059,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 14,
|
|
"sum_logprob": -21.162622451782227,
|
|
"mean_logprob": -1.5116158894130163,
|
|
"total_sequence_tokens": 22
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.10110154833112439,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03249907099962002
|
|
},
|
|
{
|
|
"id": "code_completion-023",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def factorial(n):\n result = 1\n for value in range(2, n + 1):\n",
|
|
"continuations": [
|
|
" result += value\n return n\n",
|
|
" result = value\n return value\n",
|
|
" result *= value\n return result\n",
|
|
" n -= value\n return n\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 10,
|
|
"sum_logprob": -27.79298210144043,
|
|
"mean_logprob": -2.779298210144043,
|
|
"total_sequence_tokens": 38
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 10,
|
|
"sum_logprob": -37.84375762939453,
|
|
"mean_logprob": -3.784375762939453,
|
|
"total_sequence_tokens": 38
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 10,
|
|
"sum_logprob": -4.943729877471924,
|
|
"mean_logprob": -0.4943729877471924,
|
|
"total_sequence_tokens": 38
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 10,
|
|
"sum_logprob": -37.693763732910156,
|
|
"mean_logprob": -3.7693763732910157,
|
|
"total_sequence_tokens": 38
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.2849252223968506,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03755112500039104
|
|
},
|
|
{
|
|
"id": "code_completion-024",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def first_index(items, target):\n for index, item in enumerate(items):\n if item == target:\n",
|
|
"continuations": [
|
|
" continue\n return 0\n",
|
|
" return item\n return target\n",
|
|
" break\n return index\n",
|
|
" return index\n return -1\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -34.801002502441406,
|
|
"mean_logprob": -4.350125312805176,
|
|
"total_sequence_tokens": 37
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -26.48795509338379,
|
|
"mean_logprob": -3.3109943866729736,
|
|
"total_sequence_tokens": 37
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -13.000073432922363,
|
|
"mean_logprob": -1.8571533475603377,
|
|
"total_sequence_tokens": 36
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 10,
|
|
"sum_logprob": -8.2440185546875,
|
|
"mean_logprob": -0.82440185546875,
|
|
"total_sequence_tokens": 39
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.0327514920915877,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0376353020001261
|
|
},
|
|
{
|
|
"id": "code_completion-025",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def merge_counts(left, right):\n result = dict(left)\n for key, value in right.items():\n",
|
|
"continuations": [
|
|
" result[key] = result.get(key, 0) + value\n return result\n",
|
|
" result[value] = key\n return left\n",
|
|
" result.clear()\n return right\n",
|
|
" key += value\n return result.values()\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 24,
|
|
"sum_logprob": -8.848027229309082,
|
|
"mean_logprob": -0.36866780122121173,
|
|
"total_sequence_tokens": 54
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 13,
|
|
"sum_logprob": -45.313819885253906,
|
|
"mean_logprob": -3.4856784527118387,
|
|
"total_sequence_tokens": 43
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 10,
|
|
"sum_logprob": -52.17894744873047,
|
|
"mean_logprob": -5.217894744873047,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 13,
|
|
"sum_logprob": -37.48686599731445,
|
|
"mean_logprob": -2.8836050767164965,
|
|
"total_sequence_tokens": 43
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.5149372754952846,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.048819237999850884
|
|
},
|
|
{
|
|
"id": "code_completion-026",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def chunks(items, size):\n",
|
|
"continuations": [
|
|
" return items[:size]\n",
|
|
" return [items[i:i + size] for i in range(0, len(items), size)]\n",
|
|
" return [size] * len(items)\n",
|
|
" return items[::size][0]\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -12.145590782165527,
|
|
"mean_logprob": -1.735084397452218,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 27,
|
|
"sum_logprob": -19.035940170288086,
|
|
"mean_logprob": -0.705034821121781,
|
|
"total_sequence_tokens": 35
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 13,
|
|
"sum_logprob": -21.432252883911133,
|
|
"mean_logprob": -1.6486348372239332,
|
|
"total_sequence_tokens": 21
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 10,
|
|
"sum_logprob": -35.489845275878906,
|
|
"mean_logprob": -3.5489845275878906,
|
|
"total_sequence_tokens": 18
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.9436000161021523,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03724283799965633
|
|
},
|
|
{
|
|
"id": "code_completion-027",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def balanced_parentheses(text):\n depth = 0\n for char in text:\n if char == \"(\":\n depth += 1\n elif char == \")\":\n depth -= 1\n if depth < 0:\n return False\n",
|
|
"continuations": [
|
|
" return True\n",
|
|
" return depth > 0\n",
|
|
" return depth == 0\n",
|
|
" return len(text) == depth\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -6.613285541534424,
|
|
"mean_logprob": -1.653321385383606,
|
|
"total_sequence_tokens": 72
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -13.789587020874023,
|
|
"mean_logprob": -1.723698377609253,
|
|
"total_sequence_tokens": 76
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 8,
|
|
"sum_logprob": -7.251691818237305,
|
|
"mean_logprob": -0.9064614772796631,
|
|
"total_sequence_tokens": 76
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 10,
|
|
"sum_logprob": -9.23131275177002,
|
|
"mean_logprob": -0.923131275177002,
|
|
"total_sequence_tokens": 78
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.01666979789733891,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.06129010799850221
|
|
},
|
|
{
|
|
"id": "code_completion-028",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def rotate_left(items, amount):\n if not items:\n return []\n",
|
|
"continuations": [
|
|
" return items[::-1]\n",
|
|
" return items[:amount]\n",
|
|
" return items + amount\n",
|
|
" amount %= len(items)\n return items[amount:] + items[:amount]\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -15.63742446899414,
|
|
"mean_logprob": -1.9546780586242676,
|
|
"total_sequence_tokens": 29
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 7,
|
|
"sum_logprob": -11.773261070251465,
|
|
"mean_logprob": -1.6818944386073522,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -18.23381996154785,
|
|
"mean_logprob": -2.6048314230782643,
|
|
"total_sequence_tokens": 28
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 23,
|
|
"sum_logprob": -23.40024757385254,
|
|
"mean_logprob": -1.0174020684283713,
|
|
"total_sequence_tokens": 44
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6644923701789809,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05138291900038894
|
|
},
|
|
{
|
|
"id": "code_completion-029",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def group_by_first_letter(words):\n groups = {}\n for word in words:\n",
|
|
"continuations": [
|
|
" groups.setdefault(word[0], []).append(word)\n return groups\n",
|
|
" groups[word] = word[0]\n return words\n",
|
|
" groups.append(word[0])\n return groups\n",
|
|
" word[0] = groups\n return groups.keys()\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 21,
|
|
"sum_logprob": -22.006641387939453,
|
|
"mean_logprob": -1.047935304187593,
|
|
"total_sequence_tokens": 47
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 16,
|
|
"sum_logprob": -43.108314514160156,
|
|
"mean_logprob": -2.6942696571350098,
|
|
"total_sequence_tokens": 42
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 14,
|
|
"sum_logprob": -30.153905868530273,
|
|
"mean_logprob": -2.153850419180734,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 16,
|
|
"sum_logprob": -55.40138626098633,
|
|
"mean_logprob": -3.4625866413116455,
|
|
"total_sequence_tokens": 42
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.105915114993141,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.03654749700035609
|
|
},
|
|
{
|
|
"id": "code_completion-030",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def moving_pairs(items):\n",
|
|
"continuations": [
|
|
" return zip(items, items)\n",
|
|
" return list(zip(items, items[1:]))\n",
|
|
" return items[::2]\n",
|
|
" return [(items[0], items[-1])]\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 9,
|
|
"sum_logprob": -19.146240234375,
|
|
"mean_logprob": -2.1273600260416665,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 14,
|
|
"sum_logprob": -15.492321968078613,
|
|
"mean_logprob": -1.1065944262913294,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 8,
|
|
"sum_logprob": -18.950725555419922,
|
|
"mean_logprob": -2.3688406944274902,
|
|
"total_sequence_tokens": 16
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 13,
|
|
"sum_logprob": -26.311683654785156,
|
|
"mean_logprob": -2.023975665752704,
|
|
"total_sequence_tokens": 21
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.9173812394613747,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04552672099998745
|
|
},
|
|
{
|
|
"id": "code_completion-031",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "def invert_mapping(mapping):\n",
|
|
"continuations": [
|
|
" return dict(mapping)\n",
|
|
" return list(mapping.items())\n",
|
|
" return {value: key for key, value in mapping.items()}\n",
|
|
" return mapping[::-1]\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 3,
|
|
"label": 2,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -16.197599411010742,
|
|
"mean_logprob": -2.3139427730015347,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 9,
|
|
"sum_logprob": -14.858826637268066,
|
|
"mean_logprob": -1.6509807374742296,
|
|
"total_sequence_tokens": 17
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 17,
|
|
"sum_logprob": -18.217885971069336,
|
|
"mean_logprob": -1.0716403512393726,
|
|
"total_sequence_tokens": 25
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -8.22993278503418,
|
|
"mean_logprob": -1.0287415981292725,
|
|
"total_sequence_tokens": 16
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.04289875311010016,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04215201799888746
|
|
},
|
|
{
|
|
"id": "code_completion-032",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def all_equal(items):\n iterator = iter(items)\n try:\n first = next(iterator)\n except StopIteration:\n",
|
|
"continuations": [
|
|
" return False\n return first == iterator\n",
|
|
" first = None\n return items == first\n",
|
|
" return []\n return len(items)\n",
|
|
" return True\n return all(item == first for item in iterator)\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 11,
|
|
"sum_logprob": -20.89051055908203,
|
|
"mean_logprob": -1.8991373235529119,
|
|
"total_sequence_tokens": 47
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 13,
|
|
"sum_logprob": -26.401443481445312,
|
|
"mean_logprob": -2.0308802678034854,
|
|
"total_sequence_tokens": 49
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 12,
|
|
"sum_logprob": -30.185457229614258,
|
|
"mean_logprob": -2.5154547691345215,
|
|
"total_sequence_tokens": 48
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 18,
|
|
"sum_logprob": -34.05173110961914,
|
|
"mean_logprob": -1.8917628394232855,
|
|
"total_sequence_tokens": 54
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.00737448412962638,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.042933908000122756
|
|
},
|
|
{
|
|
"id": "code_completion-033",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1050,
|
|
"context": "def transpose(rows):\n",
|
|
"continuations": [
|
|
" return [list(column) for column in zip(*rows)]\n",
|
|
" return rows[::-1]\n",
|
|
" return list(zip(rows))\n",
|
|
" return [sum(row) for row in rows]\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 3,
|
|
"label": 0,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 16,
|
|
"sum_logprob": -21.815868377685547,
|
|
"mean_logprob": -1.3634917736053467,
|
|
"total_sequence_tokens": 22
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 8,
|
|
"sum_logprob": -10.598787307739258,
|
|
"mean_logprob": -1.3248484134674072,
|
|
"total_sequence_tokens": 14
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 9,
|
|
"sum_logprob": -20.984830856323242,
|
|
"mean_logprob": -2.3316478729248047,
|
|
"total_sequence_tokens": 15
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 14,
|
|
"sum_logprob": -13.52786636352539,
|
|
"mean_logprob": -0.9662761688232422,
|
|
"total_sequence_tokens": 20
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.35857224464416504,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04276205599853711
|
|
},
|
|
{
|
|
"id": "code_completion-034",
|
|
"category": "code_completion",
|
|
"difficulty": "medium",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 1.5,
|
|
"point_weight": 2.0999999999999996,
|
|
"earned_points": 2.0999999999999996,
|
|
"item_elo": 1050,
|
|
"context": "def running_totals(numbers):\n totals = []\n current = 0\n for number in numbers:\n",
|
|
"continuations": [
|
|
" totals.append(number)\n return current\n",
|
|
" current += number\n totals.append(current)\n return totals\n",
|
|
" current = number\n return [current]\n",
|
|
" totals += current\n return numbers\n"
|
|
],
|
|
"tags": [
|
|
"python"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 13,
|
|
"sum_logprob": -39.600975036621094,
|
|
"mean_logprob": -3.0462288489708533,
|
|
"total_sequence_tokens": 44
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 20,
|
|
"sum_logprob": -6.991767883300781,
|
|
"mean_logprob": -0.34958839416503906,
|
|
"total_sequence_tokens": 51
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 13,
|
|
"sum_logprob": -40.40708923339844,
|
|
"mean_logprob": -3.1082376333383412,
|
|
"total_sequence_tokens": 44
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 11,
|
|
"sum_logprob": -54.728126525878906,
|
|
"mean_logprob": -4.975284229625355,
|
|
"total_sequence_tokens": 42
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.6966404548058143,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.057458866000160924
|
|
},
|
|
{
|
|
"id": "code_completion-035",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def binary_search(items, target):\n low, high = 0, len(items) - 1\n while low <= high:\n mid = (low + high) // 2\n if items[mid] == target:\n return mid\n if items[mid] < target:\n low = mid + 1\n else:\n",
|
|
"continuations": [
|
|
" low = mid - 1\n return low\n",
|
|
" high = mid + 1\n return high\n",
|
|
" high = mid - 1\n return -1\n",
|
|
" return mid\n return None\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"algorithm"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 14,
|
|
"sum_logprob": -36.48191452026367,
|
|
"mean_logprob": -2.605851037161691,
|
|
"total_sequence_tokens": 106
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 14,
|
|
"sum_logprob": -46.85424041748047,
|
|
"mean_logprob": -3.3467314583914622,
|
|
"total_sequence_tokens": 106
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 16,
|
|
"sum_logprob": -7.23041296005249,
|
|
"mean_logprob": -0.45190081000328064,
|
|
"total_sequence_tokens": 108
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 8,
|
|
"sum_logprob": -23.18140983581543,
|
|
"mean_logprob": -2.8976762294769287,
|
|
"total_sequence_tokens": 100
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.1539502271584103,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.07552583399956347
|
|
},
|
|
{
|
|
"id": "code_completion-036",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def merge_sorted(left, right):\n result = []\n i = j = 0\n while i < len(left) and j < len(right):\n if left[i] <= right[j]:\n result.append(left[i])\n i += 1\n else:\n result.append(right[j])\n j += 1\n",
|
|
"continuations": [
|
|
" return result\n",
|
|
" result.extend(left)\n return result\n",
|
|
" return sorted(left)\n",
|
|
" result.extend(left[i:])\n result.extend(right[j:])\n return result\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"algorithm"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 4,
|
|
"sum_logprob": -11.00175666809082,
|
|
"mean_logprob": -2.750439167022705,
|
|
"total_sequence_tokens": 100
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 12,
|
|
"sum_logprob": -27.22464370727539,
|
|
"mean_logprob": -2.2687203089396157,
|
|
"total_sequence_tokens": 108
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -48.96931457519531,
|
|
"mean_logprob": -6.995616367885044,
|
|
"total_sequence_tokens": 103
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 24,
|
|
"sum_logprob": -6.3913092613220215,
|
|
"mean_logprob": -0.26630455255508423,
|
|
"total_sequence_tokens": 120
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.0024157563845315,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.07246739999936835
|
|
},
|
|
{
|
|
"id": "code_completion-037",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def topological_visit(node, graph, visiting, visited, order):\n if node in visiting:\n raise ValueError(\"cycle\")\n if node in visited:\n return\n visiting.add(node)\n for neighbor in graph.get(node, []):\n topological_visit(neighbor, graph, visiting, visited, order)\n",
|
|
"continuations": [
|
|
" visiting.remove(node)\n visited.add(node)\n order.append(node)\n",
|
|
" order.append(node)\n visiting.add(node)\n",
|
|
" visited.remove(node)\n order.clear()\n",
|
|
" return graph[node]\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"graph"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 24,
|
|
"sum_logprob": -22.875579833984375,
|
|
"mean_logprob": -0.953149159749349,
|
|
"total_sequence_tokens": 103
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 16,
|
|
"sum_logprob": -39.46855926513672,
|
|
"mean_logprob": -2.466784954071045,
|
|
"total_sequence_tokens": 95
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 14,
|
|
"sum_logprob": -21.016754150390625,
|
|
"mean_logprob": -1.5011967250279017,
|
|
"total_sequence_tokens": 93
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 7,
|
|
"sum_logprob": -17.03171157836914,
|
|
"mean_logprob": -2.4331016540527344,
|
|
"total_sequence_tokens": 86
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5480475652785527,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.07073609600047348
|
|
},
|
|
{
|
|
"id": "code_completion-038",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "def longest_common_prefix(words):\n if not words:\n return \"\"\n prefix = words[0]\n for word in words[1:]:\n while not word.startswith(prefix):\n",
|
|
"continuations": [
|
|
" prefix += word\n",
|
|
" prefix = prefix[:-1]\n if not prefix:\n return \"\"\n return prefix\n",
|
|
" word = word[:-1]\n return word\n",
|
|
" return words[0]\n return prefix + word\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"string"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -3.5768799781799316,
|
|
"mean_logprob": -0.5961466630299886,
|
|
"total_sequence_tokens": 58
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 25,
|
|
"sum_logprob": -27.742603302001953,
|
|
"mean_logprob": -1.1097041320800782,
|
|
"total_sequence_tokens": 77
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 14,
|
|
"sum_logprob": -47.44060516357422,
|
|
"mean_logprob": -3.3886146545410156,
|
|
"total_sequence_tokens": 66
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 14,
|
|
"sum_logprob": -72.98765563964844,
|
|
"mean_logprob": -5.213403974260602,
|
|
"total_sequence_tokens": 66
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5135574690500896,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.06210656999974162
|
|
},
|
|
{
|
|
"id": "code_completion-039",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def memoized_fibonacci(n, cache=None):\n if cache is None:\n cache = {0: 0, 1: 1}\n if n not in cache:\n",
|
|
"continuations": [
|
|
" cache[n] = n\n",
|
|
" return cache\n",
|
|
" cache[n] = memoized_fibonacci(n - 1, cache) + memoized_fibonacci(n - 2, cache)\n return cache[n]\n",
|
|
" cache.clear()\n return n\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"recursion"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 9,
|
|
"sum_logprob": -11.39099407196045,
|
|
"mean_logprob": -1.2656660079956055,
|
|
"total_sequence_tokens": 56
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 4,
|
|
"sum_logprob": -30.46306037902832,
|
|
"mean_logprob": -7.61576509475708,
|
|
"total_sequence_tokens": 51
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 45,
|
|
"sum_logprob": -11.498723983764648,
|
|
"mean_logprob": -0.2555271996392144,
|
|
"total_sequence_tokens": 92
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 10,
|
|
"sum_logprob": -50.775535583496094,
|
|
"mean_logprob": -5.077553558349609,
|
|
"total_sequence_tokens": 57
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.0101388083563911,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0655426350003836
|
|
},
|
|
{
|
|
"id": "code_completion-040",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def intervals_overlap(a_start, a_end, b_start, b_end):\n",
|
|
"continuations": [
|
|
" return a_end < b_start\n",
|
|
" return a_start == b_end\n",
|
|
" return a_start + b_start\n",
|
|
" return max(a_start, b_start) <= min(a_end, b_end)\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"interval"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 11,
|
|
"sum_logprob": -16.90646743774414,
|
|
"mean_logprob": -1.5369515852494673,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 11,
|
|
"sum_logprob": -21.989330291748047,
|
|
"mean_logprob": -1.9990300265225498,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 11,
|
|
"sum_logprob": -18.211776733398438,
|
|
"mean_logprob": -1.6556160666725852,
|
|
"total_sequence_tokens": 33
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 25,
|
|
"sum_logprob": -22.111881256103516,
|
|
"mean_logprob": -0.8844752502441406,
|
|
"total_sequence_tokens": 47
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.6524763350053266,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04035365600066143
|
|
},
|
|
{
|
|
"id": "code_completion-041",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def breadth_first_distances(graph, start):\n from collections import deque\n distances = {start: 0}\n queue = deque([start])\n while queue:\n node = queue.popleft()\n for neighbor in graph.get(node, []):\n if neighbor not in distances:\n",
|
|
"continuations": [
|
|
" distances[neighbor] = distances[node] + 1\n queue.append(neighbor)\n return distances\n",
|
|
" distances[node] += 1\n return queue\n",
|
|
" queue.append(node)\n return graph\n",
|
|
" return neighbor\n return distances[start]\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"graph"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 28,
|
|
"sum_logprob": -6.422842025756836,
|
|
"mean_logprob": -0.2293872152056013,
|
|
"total_sequence_tokens": 107
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 14,
|
|
"sum_logprob": -47.448028564453125,
|
|
"mean_logprob": -3.3891448974609375,
|
|
"total_sequence_tokens": 93
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 12,
|
|
"sum_logprob": -68.20989990234375,
|
|
"mean_logprob": -5.6841583251953125,
|
|
"total_sequence_tokens": 91
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 11,
|
|
"sum_logprob": -64.69454956054688,
|
|
"mean_logprob": -5.881322687322443,
|
|
"total_sequence_tokens": 90
|
|
}
|
|
],
|
|
"best_vs_second_margin": 3.159757682255336,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.06932938600039051
|
|
},
|
|
{
|
|
"id": "code_completion-042",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def deep_get(mapping, path, default=None):\n current = mapping\n for key in path:\n if not isinstance(current, dict) or key not in current:\n return default\n",
|
|
"continuations": [
|
|
" current = mapping[key]\n return mapping\n",
|
|
" current = current[key]\n return current\n",
|
|
" path = current\n return key\n",
|
|
" return current.get(path)\n return default\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"nested_data"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 13,
|
|
"sum_logprob": -32.41277313232422,
|
|
"mean_logprob": -2.4932902409480167,
|
|
"total_sequence_tokens": 61
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 13,
|
|
"sum_logprob": -4.214959621429443,
|
|
"mean_logprob": -0.32422766318688023,
|
|
"total_sequence_tokens": 61
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 10,
|
|
"sum_logprob": -39.24943923950195,
|
|
"mean_logprob": -3.9249439239501953,
|
|
"total_sequence_tokens": 58
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 13,
|
|
"sum_logprob": -58.77112579345703,
|
|
"mean_logprob": -4.520855830265925,
|
|
"total_sequence_tokens": 61
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.1690625777611365,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04932143499900121
|
|
},
|
|
{
|
|
"id": "code_completion-043",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def stable_partition(items, predicate):\n matching = []\n other = []\n for item in items:\n if predicate(item):\n matching.append(item)\n else:\n other.append(item)\n",
|
|
"continuations": [
|
|
" return sorted(items)\n",
|
|
" return matching or other\n",
|
|
" return matching + other\n",
|
|
" return zip(matching, other)\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"algorithm"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 7,
|
|
"sum_logprob": -30.302295684814453,
|
|
"mean_logprob": -4.328899383544922,
|
|
"total_sequence_tokens": 65
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 6,
|
|
"sum_logprob": -17.109447479248047,
|
|
"mean_logprob": -2.8515745798746743,
|
|
"total_sequence_tokens": 64
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 7,
|
|
"sum_logprob": -12.190431594848633,
|
|
"mean_logprob": -1.7414902278355189,
|
|
"total_sequence_tokens": 65
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 9,
|
|
"sum_logprob": -18.88239288330078,
|
|
"mean_logprob": -2.0980436537000866,
|
|
"total_sequence_tokens": 67
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.35655342586456773,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05759788499926799
|
|
},
|
|
{
|
|
"id": "code_completion-044",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def parse_key_value(lines):\n result = {}\n for line in lines:\n line = line.strip()\n if not line or line.startswith(\"#\"):\n continue\n key, separator, value = line.partition(\"=\")\n if not separator:\n raise ValueError(\"missing equals sign\")\n",
|
|
"continuations": [
|
|
" result[value] = separator\n return lines\n",
|
|
" result[key] = key\n return value\n",
|
|
" result.clear()\n return key\n",
|
|
" result[key.strip()] = value.strip()\n return result\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"parsing"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 13,
|
|
"sum_logprob": -49.231327056884766,
|
|
"mean_logprob": -3.787025158221905,
|
|
"total_sequence_tokens": 94
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 13,
|
|
"sum_logprob": -42.809051513671875,
|
|
"mean_logprob": -3.293003962590144,
|
|
"total_sequence_tokens": 94
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 10,
|
|
"sum_logprob": -46.267005920410156,
|
|
"mean_logprob": -4.626700592041016,
|
|
"total_sequence_tokens": 91
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 18,
|
|
"sum_logprob": -8.798966407775879,
|
|
"mean_logprob": -0.48883146709865993,
|
|
"total_sequence_tokens": 99
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.804172495491484,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.07062074500026938
|
|
},
|
|
{
|
|
"id": "code_completion-045",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def sliding_window_sums(numbers, width):\n if width <= 0 or width > len(numbers):\n return []\n current = sum(numbers[:width])\n result = [current]\n for index in range(width, len(numbers)):\n",
|
|
"continuations": [
|
|
" current += numbers[index] - numbers[index - width]\n result.append(current)\n return result\n",
|
|
" current = numbers[index]\n return [current]\n",
|
|
" result.append(sum(numbers))\n return numbers\n",
|
|
" width += index\n return result[:width]\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"algorithm"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 30,
|
|
"sum_logprob": -41.30909729003906,
|
|
"mean_logprob": -1.3769699096679688,
|
|
"total_sequence_tokens": 98
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 16,
|
|
"sum_logprob": -62.10095977783203,
|
|
"mean_logprob": -3.881309986114502,
|
|
"total_sequence_tokens": 84
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 14,
|
|
"sum_logprob": -73.57785034179688,
|
|
"mean_logprob": -5.255560738699777,
|
|
"total_sequence_tokens": 82
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 13,
|
|
"sum_logprob": -64.48774719238281,
|
|
"mean_logprob": -4.960595937875601,
|
|
"total_sequence_tokens": 81
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.504340076446533,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.07155449200035946
|
|
},
|
|
{
|
|
"id": "code_completion-046",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 0.0,
|
|
"item_elo": 1200,
|
|
"context": "def retry(operation, attempts):\n last_error = None\n for _ in range(attempts):\n try:\n return operation()\n except Exception as error:\n last_error = error\n",
|
|
"continuations": [
|
|
" return last_error\n",
|
|
" if last_error is not None:\n raise last_error\n raise ValueError(\"attempts must be positive\")\n",
|
|
" operation()\n return None\n",
|
|
" except last_error:\n return attempts\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"exceptions"
|
|
],
|
|
"prediction": 0,
|
|
"label": 1,
|
|
"passed": false,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 6,
|
|
"sum_logprob": -7.404885768890381,
|
|
"mean_logprob": -1.2341476281483967,
|
|
"total_sequence_tokens": 57
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 28,
|
|
"sum_logprob": -61.165992736816406,
|
|
"mean_logprob": -2.184499740600586,
|
|
"total_sequence_tokens": 79
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 8,
|
|
"sum_logprob": -27.670310974121094,
|
|
"mean_logprob": -3.4587888717651367,
|
|
"total_sequence_tokens": 59
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 11,
|
|
"sum_logprob": -44.15677261352539,
|
|
"mean_logprob": -4.014252055775035,
|
|
"total_sequence_tokens": 62
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.9503521124521892,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.05811959799939359
|
|
},
|
|
{
|
|
"id": "code_completion-047",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def normalize_rows(matrix):\n result = []\n for row in matrix:\n total = sum(row)\n if total == 0:\n result.append([0.0 for _ in row])\n else:\n",
|
|
"continuations": [
|
|
" result.append(row)\n",
|
|
" result.append([total for _ in row])\n",
|
|
" result.append([value / total for value in row])\n return result\n",
|
|
" result = row / total\n return matrix\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"numeric"
|
|
],
|
|
"prediction": 2,
|
|
"label": 2,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 8,
|
|
"sum_logprob": -12.316730499267578,
|
|
"mean_logprob": -1.5395913124084473,
|
|
"total_sequence_tokens": 67
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 13,
|
|
"sum_logprob": -21.836929321289062,
|
|
"mean_logprob": -1.6797637939453125,
|
|
"total_sequence_tokens": 72
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 19,
|
|
"sum_logprob": -18.89575958251953,
|
|
"mean_logprob": -0.9945136622378701,
|
|
"total_sequence_tokens": 78
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 13,
|
|
"sum_logprob": -36.02602005004883,
|
|
"mean_logprob": -2.7712323115422177,
|
|
"total_sequence_tokens": 72
|
|
}
|
|
],
|
|
"best_vs_second_margin": 0.5450776501705772,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.04855151400079194
|
|
},
|
|
{
|
|
"id": "code_completion-048",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def merge_ranges(ranges):\n if not ranges:\n return []\n ordered = sorted(ranges)\n merged = [list(ordered[0])]\n for start, end in ordered[1:]:\n previous = merged[-1]\n if start <= previous[1]:\n previous[1] = max(previous[1], end)\n else:\n",
|
|
"continuations": [
|
|
" previous[0] = start\n return ranges\n",
|
|
" merged = [[start, end]]\n return ordered\n",
|
|
" return previous\n return merged\n",
|
|
" merged.append([start, end])\n return [tuple(pair) for pair in merged]\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"interval"
|
|
],
|
|
"prediction": 3,
|
|
"label": 3,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 13,
|
|
"sum_logprob": -77.74765014648438,
|
|
"mean_logprob": -5.98058847280649,
|
|
"total_sequence_tokens": 105
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 15,
|
|
"sum_logprob": -66.21233367919922,
|
|
"mean_logprob": -4.414155578613281,
|
|
"total_sequence_tokens": 107
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 8,
|
|
"sum_logprob": -38.67328643798828,
|
|
"mean_logprob": -4.834160804748535,
|
|
"total_sequence_tokens": 100
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 24,
|
|
"sum_logprob": -42.53789520263672,
|
|
"mean_logprob": -1.7724123001098633,
|
|
"total_sequence_tokens": 116
|
|
}
|
|
],
|
|
"best_vs_second_margin": 2.6417432785034176,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.07347782599936181
|
|
},
|
|
{
|
|
"id": "code_completion-049",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def count_paths(grid, row=0, col=0, memo=None):\n if memo is None:\n memo = {}\n if row >= len(grid) or col >= len(grid[0]) or grid[row][col] == 1:\n return 0\n if (row, col) == (len(grid) - 1, len(grid[0]) - 1):\n return 1\n if (row, col) not in memo:\n",
|
|
"continuations": [
|
|
" memo[(row, col)] = count_paths(grid, row + 1, col, memo) + count_paths(grid, row, col + 1, memo)\n return memo[(row, col)]\n",
|
|
" memo.clear()\n return 0\n",
|
|
" return count_paths(grid, row - 1, col - 1, memo)\n return memo\n",
|
|
" memo[(row, col)] = 1\n return len(memo)\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"dynamic_programming"
|
|
],
|
|
"prediction": 0,
|
|
"label": 0,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 53,
|
|
"sum_logprob": -6.174004554748535,
|
|
"mean_logprob": -0.11649065197638746,
|
|
"total_sequence_tokens": 175
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 11,
|
|
"sum_logprob": -51.11182403564453,
|
|
"mean_logprob": -4.646529457785866,
|
|
"total_sequence_tokens": 133
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 27,
|
|
"sum_logprob": -71.41888427734375,
|
|
"mean_logprob": -2.6451438621238426,
|
|
"total_sequence_tokens": 149
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 20,
|
|
"sum_logprob": -37.429500579833984,
|
|
"mean_logprob": -1.8714750289916993,
|
|
"total_sequence_tokens": 142
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.7549843770153117,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.10541650500090327
|
|
},
|
|
{
|
|
"id": "code_completion-050",
|
|
"category": "code_completion",
|
|
"difficulty": "hard",
|
|
"category_weight": 1.4,
|
|
"difficulty_weight": 2.25,
|
|
"point_weight": 3.15,
|
|
"earned_points": 3.15,
|
|
"item_elo": 1200,
|
|
"context": "def compose(*functions):\n def composed(value):\n result = value\n for function in reversed(functions):\n",
|
|
"continuations": [
|
|
" result = functions(result)\n return value\n",
|
|
" result = function(result)\n return result\n return composed\n",
|
|
" return function\n return functions\n return result\n",
|
|
" result.append(function)\n return result\n return functions\n"
|
|
],
|
|
"tags": [
|
|
"python",
|
|
"higher_order"
|
|
],
|
|
"prediction": 1,
|
|
"label": 1,
|
|
"passed": true,
|
|
"choice_scores": [
|
|
{
|
|
"index": 0,
|
|
"tokens": 13,
|
|
"sum_logprob": -36.70912170410156,
|
|
"mean_logprob": -2.823778592623197,
|
|
"total_sequence_tokens": 41
|
|
},
|
|
{
|
|
"index": 1,
|
|
"tokens": 17,
|
|
"sum_logprob": -9.565998077392578,
|
|
"mean_logprob": -0.562705769258387,
|
|
"total_sequence_tokens": 45
|
|
},
|
|
{
|
|
"index": 2,
|
|
"tokens": 12,
|
|
"sum_logprob": -66.07337951660156,
|
|
"mean_logprob": -5.506114959716797,
|
|
"total_sequence_tokens": 40
|
|
},
|
|
{
|
|
"index": 3,
|
|
"tokens": 16,
|
|
"sum_logprob": -38.47551727294922,
|
|
"mean_logprob": -2.404719829559326,
|
|
"total_sequence_tokens": 44
|
|
}
|
|
],
|
|
"best_vs_second_margin": 1.8420140603009392,
|
|
"context_truncated": false,
|
|
"elapsed_seconds": 0.0417035800001031
|
|
}
|
|
]
|
|
}
|