Solshine/Llama-3.2-1B-sandbag-circuit-ablated
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
Llama-3.2-1B-sandbag-circui…/benchmarks/mmlu_base.json
ModelHub XC 1f3458b152 初始化项目，由ModelHub XC社区提供模型 
Model: Solshine/Llama-3.2-1B-sandbag-circuit-ablated
Source: Original Platform
2026-05-22 11:49:16 +08:00

4861 lines
181 KiB
JSON
Raw Permalink Blame History

{
"model": "meta-llama/Llama-3.2-1B",
"device": "cpu",
"dtype": "float32",
"task": "mmlu_0shot_letter_likelihood",
"n_questions": 228,
"seed": 17,
"n_evaluated": 228,
"n_correct": 90,
"accuracy": 0.39473684210526316,
"per_item": [
{
"ds_idx": 2450,
"subject": "formal_logic",
"question": "Identify the antecedent of the following conditional proposition: If the university does not increase financial aid, either the president fails to approve it or the board of trustees prevents it.",
"choices": [
"The university increases financial aid.",
"The university does not increase financial aid.",
"The board of trustees prevents it.",
"The president fails to approve it."
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.363113522529602,
"B": -1.124233365058899,
"C": -1.5823174715042114,
"D": -1.5704976320266724
}
},
{
"ds_idx": 3977,
"subject": "high_school_macroeconomics",
"question": "Which of the following is the source of the supply of loanable funds?",
"choices": [
"The stock market",
"Investors",
"Net exports",
"Savers"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.5422660112380981,
"B": -1.174002766609192,
"C": -1.57593834400177,
"D": -1.316151738166809
}
},
{
"ds_idx": 7022,
"subject": "management",
"question": "What do you call a style of leadership that takes account of others' views, opinions and ideas?",
"choices": [
"Laissez-faire",
"People-oriented",
"Democratic",
"Autocratic"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -2.2404067516326904,
"B": -0.4828365445137024,
"C": -1.9608943462371826,
"D": -2.0224311351776123
}
},
{
"ds_idx": 4741,
"subject": "high_school_physics",
"question": "A 1-kg object is released from rest at the top of a rough-surfaced incline. The object slides without rotating to the bottom of the incline. The object\u2019s kinetic energy at the bottom must be",
"choices": [
"Equal to the block\u2019s gravitational potential energy when it was released, because total mechanical energy must be conserved.",
"Equal to the block\u2019s gravitational potential energy when it was released, because the gain in kinetic energy compensates for the mechanical energy lost to thermal energy on the rough incline.",
"Less than the block\u2019s gravitational potential energy when it was released, because the gravitational potential energy was converted both to thermal energy and to kinetic energy.",
"Less than the block\u2019s gravitational potential energy when it was released, because the work done by the friction force must be greater than the block\u2019s gain in kinetic energy."
],
"correct_idx": 2,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.2177492380142212,
"B": -1.5324331521987915,
"C": -1.4350534677505493,
"D": -1.4107328653335571
}
},
{
"ds_idx": 5605,
"subject": "high_school_statistics",
"question": "A major polling organization wants to predict the outcome of an upcoming national election (in terms of the proportion of voters who will vote for each candidate). They intend to use a 95% confidence interval with margin of error of no more than 2.5%. What is the minimum sample size needed to accomplish this goal?",
"choices": [
"1536",
"39",
"1537",
"40"
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.6112563610076904,
"B": -1.2692744731903076,
"C": -1.4884707927703857,
"D": -1.2375438213348389
}
},
{
"ds_idx": 13194,
"subject": "security_studies",
"question": "How is capitalism defined or regarded within the HM paradigm?",
"choices": [
"HM regards capitalism as producing extreme insecurity in its disempowerment of the worker or labourer, from whom the ability to control labour or output is removed by the capitalist.",
"HM regards the assertion by liberalists and realists that the capitalist economic sphere is non-political to be a misrepresentation, serving the interests of the unelected capitalist elite.",
"HM regards the capitalist's accumulation of surplus value as a by-product of labour to be exploitation of those labourers by which the value was originally produced.",
"All of these options."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.4077115058898926,
"B": -1.6274304389953613,
"C": -1.9815106391906738,
"D": -0.8977741599082947
}
},
{
"ds_idx": 6633,
"subject": "jurisprudence",
"question": "On what grounds does John Finnis reject Hume's conception of practical reason?",
"choices": [
"That natural law corresponds to positive law.",
"That Hume is a legal positivist.",
"That syllogistic logic is false.",
"That human reason can help us to determine what constitutes a worthwhile life."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.3004285097122192,
"B": -1.5287991762161255,
"C": -1.933101773262024,
"D": -1.016475796699524
}
},
{
"ds_idx": 13544,
"subject": "sociology",
"question": "Which of the following is not recognised as a level of society?",
"choices": [
"the household",
"the office",
"the global village",
"the nation state"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5240532159805298,
"B": -1.3722625970840454,
"C": -1.4851585626602173,
"D": -1.2135769128799438
}
},
{
"ds_idx": 3734,
"subject": "high_school_government_and_politics",
"question": "To which characteristic of American government does the term \"federalism\" refer?",
"choices": [
"The system of checks and balances within the national government",
"The power of the Supreme Court to review the constitutionality of laws",
"The Bill of Rights' protection of the rights of the accused",
"The division and sharing of power between the national and state governments"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.877065896987915,
"B": -2.2730562686920166,
"C": -2.0932180881500244,
"D": -0.4830438494682312
}
},
{
"ds_idx": 2174,
"subject": "elementary_mathematics",
"question": "At West Elementary School, there are 20 more girls than boys. If there are 180 girls, how can you find the number of boys?",
"choices": [
"add 20 to 180",
"subtract 20 from 180",
"multiply 180 by 20",
"divide 180 by 20"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4600813388824463,
"B": -1.4289686679840088,
"C": -1.4869635105133057,
"D": -1.2438127994537354
}
},
{
"ds_idx": 38,
"subject": "abstract_algebra",
"question": "Find the generator for the finite field Z_7.",
"choices": [
"1",
"2",
"3",
"4"
],
"correct_idx": 2,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.296683430671692,
"B": -1.3108149766921997,
"C": -1.4707614183425903,
"D": -1.5100470781326294
}
},
{
"ds_idx": 4329,
"subject": "high_school_mathematics",
"question": "Let $f(x) = 3x^2-2$ and $g(f(x)) = x^2 + x +1$. Find the sum of all possible values of $g(25)$.",
"choices": [
"13",
"25",
"9",
"20"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.2958959341049194,
"B": -1.1930192708969116,
"C": -1.52877938747406,
"D": -1.642844557762146
}
},
{
"ds_idx": 7009,
"subject": "management",
"question": "Which of these factors is not one of the competing responsibilities that influence corporate social responsibility?",
"choices": [
"Ethical",
"Economic",
"Discretionary",
"Technical"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.7142561674118042,
"B": -1.2117575407028198,
"C": -1.6301974058151245,
"D": -1.1277483701705933
}
},
{
"ds_idx": 13273,
"subject": "security_studies",
"question": "Which statement best describes the relationship between realist international relations theorists and peace researchers in the early 1960s?",
"choices": [
"Both peace researchers and realists advocated an academic focus on the escalation of the arms race. Although tensions existed between the groups in terms of their proposed solutions to this crisis, there existed a consensus that the Soviet bloc should be eliminated as a nuclear threat, thus favouring the survival of Western liberal democracies.",
"In contrast to the realist school, peace researchers looked beyond the accumulation of nuclear weapons for a focus to their studies; this resulted in tensions between the two groups, with the former believing that the attainment of superior military strength was paramount to achieving security.",
"Whilst peace researchers indentified the escalation of the arms race as a result of failures by the League of Nations, realist scholars deemed it to be the natural progression of ideological differences between the Soviet bloc and Western liberal democracies. This resulted in irreconcilable differences between the study agendas of realists and peace researchers.",
"The East-West divide of the Cold War gave rise to a sharp divide between realists and early peace studies theorists. Whilst realists wished to focus research on that which would favour the survival of Western liberal democracies, advocates of peace studies regarded this approach as being narrow and Western ethnocentric."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.3578048944473267,
"B": -1.5060917139053345,
"C": -1.4801784753799438,
"D": -1.2354332208633423
}
},
{
"ds_idx": 557,
"subject": "clinical_knowledge",
"question": "Removal of adhesive strips:",
"choices": [
"can be carried out by the patient.",
"should be undertaken under aseptic conditions.",
"is a specialist skill.",
"should not be undertaken until at least seven days after application."
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4048963785171509,
"B": -1.3840337991714478,
"C": -1.3992544412612915,
"D": -1.3701025247573853
}
},
{
"ds_idx": 1981,
"subject": "electrical_engineering",
"question": "Length of the cable is doubled, its capacitance C will be",
"choices": [
"one-fourth.",
"one-half.",
"doubled.",
"unchanged."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.4673179388046265,
"B": -1.3764880895614624,
"C": -1.175115942955017,
"D": -1.6056426763534546
}
},
{
"ds_idx": 13925,
"subject": "world_religions",
"question": "Which pope promoted the cults of St. Peter and St. Paul to bolster authority in Rome?",
"choices": [
"Gregory",
"Augustine",
"Athanasius",
"Francis"
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.347280740737915,
"B": -1.066988229751587,
"C": -1.3910276889801025,
"D": -1.9496004581451416
}
},
{
"ds_idx": 4386,
"subject": "high_school_mathematics",
"question": "Morgan is on a social networking website. She has 60 female friends and 50 male friends and belongs to 5 social groups. If 30% of her female friends live in town and 40% of her male friends live in town, how many out-of-town friends does she have on the social networking site?",
"choices": [
"38",
"72",
"110",
"360"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.418330192565918,
"B": -1.2860078811645508,
"C": -1.3848543167114258,
"D": -1.486577033996582
}
},
{
"ds_idx": 7340,
"subject": "medical_genetics",
"question": "In meiosis, recombination occurs in:",
"choices": [
"Metaphase I.",
"Prophase I.",
"Metaphase II.",
"Prophase II."
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.2979679107666016,
"B": -1.2705192565917969,
"C": -1.1795063018798828,
"D": -1.9829444885253906
}
},
{
"ds_idx": 1763,
"subject": "conceptual_physics",
"question": "Carbon-14 is primarily produced by cosmic radiation in the",
"choices": [
"atmosphere",
"food we eat",
"Earth\u2019s interior",
"fallout of nuclear bomb tests"
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.7890079021453857,
"B": -1.5343081951141357,
"C": -1.045159101486206,
"D": -1.334913969039917
}
},
{
"ds_idx": 3764,
"subject": "high_school_government_and_politics",
"question": "Which of the following presidential powers is shared with the Senate?",
"choices": [
"Deploying troops",
"Drafting appropriations bills",
"Negotiating treaties",
"Forcing Congress into session"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.7976059913635254,
"B": -1.241203784942627,
"C": -1.2811150550842285,
"D": -1.331169605255127
}
},
{
"ds_idx": 13541,
"subject": "sociology",
"question": "Millions of immigrant children who entered the United States learned English and the duties of citizenship while attending public schools in the communities in which they settled. This is an example of which of the following sociological concepts?",
"choices": [
"Acculturation",
"Collective behavior",
"Social stratification",
"Social mobility"
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -0.860603928565979,
"B": -1.5290342569351196,
"C": -1.6270643472671509,
"D": -1.8281618356704712
}
},
{
"ds_idx": 13896,
"subject": "world_religions",
"question": " Which of the following shuns politics?",
"choices": [
"The sages of the Zhuangzi",
"The Songs of the South",
"The Duke of Zhou",
"The Confucian \"sage kings\""
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.348446011543274,
"B": -1.434597134590149,
"C": -1.3418446779251099,
"D": -1.445602536201477
}
},
{
"ds_idx": 5660,
"subject": "high_school_us_history",
"question": "This question refers to the following information.\n\"I observe the great and wonderful mistake, both our own and our fathers, as to the civil powers of this world, acting in spiritual matters. I have read\u2026the last will and testament of the Lord Jesus over many times, and yet I cannot find by one tittle of that testament that if He had been pleased to have accepted of a temporal crown and government that ever He would have put forth the least finger of temporal or civil power in the matters of His spiritual affairs and Kingdom. Hence must it lamentably be against the testimony of Christ Jesus for the civil state to impose upon the souls of the people a religion, a worship, a ministry, oaths (in religious and civil affairs), tithes, times, days, marryings, and buryings in holy ground\u2026\"\nRoger Williams, The Hireling Ministry None of Christ's, 1652\nThe Puritans believed that the freedom to practice religion should be extended to",
"choices": [
"Puritans only",
"all Protestants only",
"all Christians only",
"all Jews and Christians only"
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.7117277383804321,
"B": -1.7454286813735962,
"C": -0.9183050394058228,
"D": -1.4191843271255493
}
},
{
"ds_idx": 2664,
"subject": "global_facts",
"question": "As of 2017, the share of global deaths by suicide is about",
"choices": [
"0.70%",
"1.40%",
"2.80%",
"5.60%"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5808119773864746,
"B": -1.5092253684997559,
"C": -1.2639479637145996,
"D": -1.25050687789917
}
},
{
"ds_idx": 12447,
"subject": "professional_psychology",
"question": "A researcher interested in examining the potential impact of parent alcoholism on child and family development recruits 12-year-olds (n = 100), 13-year-olds (n = 100), and 14-year-olds (n = 100)\u2014half of whom have an alcoholic parent and half of whom do not\u2014into a multiple-year longitudinal study assessing various outcomes. This study is best characterized as:",
"choices": [
"A true experiment",
"A cross-sequential cohort design",
"A natural experiment",
"A cross-sectional cohort design"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.5725958347320557,
"B": -1.8109800815582275,
"C": -0.6927072405815125,
"D": -2.0643389225006104
}
},
{
"ds_idx": 1473,
"subject": "computer_security",
"question": "MIT\u2019s Kerberos KDC server has a maximum ticket lifetime of 24 hours (for most user principals). What ensures that an expired Kerberos ticket can no longer be used?",
"choices": [
"The Kerberos server (KDC) refuses to establish new connections between clients and servers for expired tickets.",
"When a client connects to a server, the server sets a 24-hour timer to terminate the connection, which ensures a client cannot remain connected past the ticket\u2019s maximum lifetime.",
"When a client connects to a server, the server compares the ticket\u2019s expiration time to the server\u2019s current clock, and refuses to authenticate the user if the ticket expiration time is in the past.",
"When a client connects to a server, the server sends a query to the KDC to check if the ticket is still valid with respect to the KDC\u2019s clock, and refuses to authenticate the user if the KDC reports that the ticket is expired."
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5682663917541504,
"B": -1.3974308967590332,
"C": -1.6617207527160645,
"D": -1.0588974952697754
}
},
{
"ds_idx": 13628,
"subject": "us_foreign_policy",
"question": "What led Britain to impose new taxes on their American colonies?",
"choices": [
"To increase the wealth of King George III",
"The growing costs of war with France",
"Anger at America's growing prosperity",
"Pressure from rich merchants"
],
"correct_idx": 1,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.120222806930542,
"B": -1.4017970561981201,
"C": -1.4242522716522217,
"D": -1.7002437114715576
}
},
{
"ds_idx": 1160,
"subject": "college_mathematics",
"question": "Which of the following is NOT a group?",
"choices": [
"The integers under addition",
"The nonzero integers under multiplication",
"The nonzero real numbers under multiplication",
"The complex numbers under addition"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.6092798709869385,
"B": -1.3388235569000244,
"C": -1.3876383304595947,
"D": -1.2778551578521729
}
},
{
"ds_idx": 3147,
"subject": "high_school_chemistry",
"question": "Nitrous acid is a weak acid, while nitric acid is a much stronger acid because",
"choices": [
"the nitrogen in nitric acid is more electronegative than the nitrogen in nitrous acid",
"nitric acid has more oxygen atoms in its formula",
"the \u2013O\u2013H bonds in nitric acid are much weaker than in nitrous acid due to the electron withdrawing of more oxygen atoms on nitric acid",
"nitric acid has the hydrogen atoms bound directly to the nitrogen atom"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.2179651260375977,
"B": -1.5153589248657227,
"C": -1.0974855422973633,
"D": -1.9430856704711914
}
},
{
"ds_idx": 13108,
"subject": "public_relations",
"question": "Which traditional news criterion would you expect to be most important in public relations messages distributed through the mass media?",
"choices": [
"Proximity",
"Timeliness",
"Prominence",
"Impact"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.7087256908416748,
"B": -1.0192134380340576,
"C": -1.436852216720581,
"D": -1.5213820934295654
}
},
{
"ds_idx": 4681,
"subject": "high_school_microeconomics",
"question": "Which of the following is true for a firm operating under perfect competition in the long run?",
"choices": [
"The firm will engage in non-price competition.",
"The firm will realize economic profits.",
"The firm will incur economic losses.",
"The firm will realize zero economic profits."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.4593032598495483,
"B": -1.503689169883728,
"C": -1.4555667638778687,
"D": -1.1779597997665405
}
},
{
"ds_idx": 7854,
"subject": "miscellaneous",
"question": "Students are reading the following passage from the National Park Service's Web site. Ten miles long and nearly 500 feet deep (152.4 m meters), Lake McDonald, the largest lake in the park, is a direct result of glacial carving. High peaks surrounding the lake all show evidence of the power of glaciers to carve even the hardest of rock. The powerful glaciers that carved the broad U-shaped valley that Lake McDonald sits in also carved smaller hanging valleys with wonderful waterfalls that are accessible by numerous hiking trails. Which of the following best identifies the intended audience of the passage?",
"choices": [
"Scientists researching glacier development",
"Teachers interested in creating virtual park field trips",
"Academics studying the historical context of national parks",
"Families interested in visiting Lake McDonald"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.5297305583953857,
"B": -1.3770778179168701,
"C": -1.2592494487762451,
"D": -1.4019935131072998
}
},
{
"ds_idx": 13642,
"subject": "us_foreign_policy",
"question": "The trade-off between defense and social spending is often referred to as a choice between",
"choices": [
"bread and butter.",
"war and peace.",
"guns and butter.",
"bombs and books."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.577082633972168,
"B": -1.2573575973510742,
"C": -1.0010194778442383,
"D": -1.9977540969848633
}
},
{
"ds_idx": 13519,
"subject": "sociology",
"question": "The term 'feminist standpoint' suggests:",
"choices": [
"taking a stand on the issues neglected by feminism",
"studying society from the perspective of women",
"the recognition of difference and diversity in women's lives",
"a tendency to ignore the gendered nature of knowledge"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.774983525276184,
"B": -1.0166999101638794,
"C": -1.2999526262283325,
"D": -1.6420928239822388
}
},
{
"ds_idx": 3306,
"subject": "high_school_european_history",
"question": "This question refers to the following information.\n\"The Government of the German Reich and The Government of the Union of Soviet Socialist Republics desirous of strengthening the cause of peace between Germany and the U.S.S.R., and proceeding from the fundamental provisions of the Neutrality Agreement concluded in April, 1926 between Germany and the U.S.S.R., have reached the following Agreement:\nArticle I. Both High Contracting Parties obligate themselves to desist from any act of violence, any aggressive action, and any attack on each other, either individually or jointly with other Powers.\nArticle II. Should one of the High Contracting Parties become the object of belligerent action by a third Power, the other High Contracting Party shall in no manner lend its support to this third Power.\nArticle III. The Governments of the two High Contracting Parties shall in the future maintain continual contact with one another for the purpose of consultation in order to exchange information on problems affecting their common interests.\nArticle IV. Should disputes or conflicts arise between the High Contracting Parties shall participate in any grouping of Powers whatsoever that is directly or indirectly aimed at the other party.\nArticle V. Should disputes or conflicts arise between the High Contracting Parties over problems of one kind or another, both parties shall settle these disputes or conflicts exclusively through friendly exchange of opinion or, if necessary, through the establishment of arbitration commissions.\"\nMolotov-Ribbentrop Pact, 1939\nThis agreement allowed both nations involved to freely invade which country?",
"choices": [
"Denmark",
"Finland",
"France",
"Poland"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.7633111476898193,
"B": -1.5579984188079834,
"C": -1.6193883419036865,
"D": -0.8796555995941162
}
},
{
"ds_idx": 5702,
"subject": "high_school_us_history",
"question": "This question refers to the following information.\nOn Being Brought from Africa to America\n'Twas mercy brought me from my Pagan land,\nTaught my benighted soul to understand\nThat there's a God, that there's a Saviour too;\nOnce I redemption neither sought nor knew.\nSome view our sable race with scornful eye,\n\"Their colour is a diabolic die.\"\nRemember, Christians, Negroes, black as Cain,\nMay be refin'd, and join th' angelic train.\n\u2014Phillis Wheatley, Poems on Various Subjects, Religious and Moral, 1773\nThe point of Wheatley's poem can best be compared to which of the following?",
"choices": [
"The Declaration of Independence",
"Jonathan Edwards's sermon \"Sinners in the Hands of an Angry God\"",
"The Seneca Falls Declaration of Rights and Sentiments",
"Martin Luther King, Jr.'s \"I Have a Dream\u201d speech"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.5995817184448242,
"B": -1.2787141799926758,
"C": -1.395737648010254,
"D": -1.3118829727172852
}
},
{
"ds_idx": 1518,
"subject": "computer_security",
"question": "Which of the following is not a block cipher operating mode?",
"choices": [
"ECB",
"CFB",
"CBF",
"CBC"
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4346520900726318,
"B": -1.387573003768921,
"C": -1.509155035018921,
"D": -1.2586991786956787
}
},
{
"ds_idx": 144,
"subject": "anatomy",
"question": "The mucosa of the upper larynx is innervated by the",
"choices": [
"internal laryngeal nerve which is the afferent limb of the cough reflex.",
"external laryngeal nerve which is the afferent limb of the gag reflex.",
"internal laryngeal nerve which is the afferent limb of the gag reflex.",
"external laryngeal nerve which is the afferent limb of the cough reflex."
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -1.2392570972442627,
"B": -1.447547197341919,
"C": -1.5178310871124268,
"D": -1.379725694656372
}
},
{
"ds_idx": 13791,
"subject": "virology",
"question": "Identify the most commonly used diagnostic technique in the virus laboratory:",
"choices": [
"Virus isolation in cell culture",
"Deep sequencing (NGS)",
"Reverse transcriptase (RT) PCR",
"Immuno-serology"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.0620373487472534,
"B": -1.9538735151290894,
"C": -0.8188999891281128,
"D": -2.66988468170166
}
},
{
"ds_idx": 10143,
"subject": "prehistory",
"question": "Based on the morphology of fossil hominids dating from about 1.8 million until about 400,000 years ago, researchers argue that:",
"choices": [
"there were two or more related but geographically separate hominid species during this period that evolved from earlier African hominids",
"hominids dating from this time can be grouped into different species in Africa, Asia, and Europe.",
"all of the hominids that evolved from Homo habilis and predate Homo sapiens belong in a single, geographically widespread but morphologically diverse species known as Homo erectus.",
"all of the above"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.6105997562408447,
"B": -1.9800989627838135,
"C": -1.8625719547271729,
"D": -0.6846602559089661
}
},
{
"ds_idx": 7039,
"subject": "management",
"question": "Of what is advertising a form?",
"choices": [
"Focusing strategy",
"Differentiation",
"Cost leadership",
"Growth"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4646506309509277,
"B": -1.3361754417419434,
"C": -1.073840618133545,
"D": -1.82826566696167
}
},
{
"ds_idx": 9800,
"subject": "philosophy",
"question": "Epictetus recommends that we refrain from:",
"choices": [
"laughing without restraint.",
"casually going to lectures.",
"having sex before marriage.",
"all of the above."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.4794012308120728,
"B": -1.5242868661880493,
"C": -1.5124536752700806,
"D": -1.1083303689956665
}
},
{
"ds_idx": 3351,
"subject": "high_school_european_history",
"question": "This question refers to the following information.\nConfidential\u2014For Your Excellency's personal information and guidance\nThe Austro-Hungarian Ambassador yesterday delivered to the [German] Emperor [Wilhelm II] a confidential personal letter from the Emperor Francis Joseph [of Austria-Hungary], which depicts the present situation from the Austro-Hungarian point of view, and describes the measures which Vienna has in view. A copy is now being forwarded to Your Excellency. \u2026\nHis Majesty desires to say that he is not blind to the danger which threatens Austria-Hungary and thus the Triple Alliance as a result of the Russian and Serbian Pan-Slavic agitation. \u2026 His Majesty will, furthermore, make an effort at Bucharest, according to the wishes of the Emperor Franz Joseph, to influence King Carol to the fulfilment of the duties of his alliance, to the renunciation of Serbia, and to the suppression of the Rumanian agitations directed against Austria-Hungary.\nFinally, as far as concerns Serbia, His Majesty, of course, cannot interfere in the dispute now going on between Austria-Hungary and that country, as it is a matter not within his competence. The Emperor Franz Joseph may, however, rest assured that His Majesty will faithfully stand by Austria-Hungary, as is required by the obligations of his alliance and of his ancient friendship.\nTheobald von Bethmann-Hollweg (chancellor of Germany), telegram to the German ambassador at Vienna, July 6, 1914\nFrom the passage, one may infer that Bethmann-Hollweg's telegram is often referred to as Germany's \"blank check\" because",
"choices": [
"it pledged Germany to join the Triple Alliance and support Austria-Hungary against the Triple Entente",
"it was understood to give Austria an unlimited scope of response to the assassination of Franz Ferdinand, to Serbia, and Pan-Slavism within the Austro-Hungarian Empire",
"it pledged Germany's unlimited support to Franz Joseph in his efforts to succeed the recently assassinated Franz Ferdinand",
"it offered nothing in terms of real support to Franz Joseph and Austria-Hungary"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4658873081207275,
"B": -1.374483346939087,
"C": -1.4224035739898682,
"D": -1.3537828922271729
}
},
{
"ds_idx": 888,
"subject": "college_biology",
"question": "A female fruit fly bearing linked genes that produce the phenotype gray body and normal wings mates with a male fruit fly of phenotype black body and vestigial wings. The presence of gray-bodied, vestigialwinged flies among the progeny is best explained by",
"choices": [
"crossing over",
"independent assortment",
"segregation of alleles",
"penetrance"
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.4322985410690308,
"B": -0.9890345335006714,
"C": -1.6271113157272339,
"D": -1.6600836515426636
}
},
{
"ds_idx": 1812,
"subject": "econometrics",
"question": "A parsimonious model is one that",
"choices": [
"Includes too many variables",
"Includes as few variables as possible to explain the data",
"Is a well-specified model",
"Is a mis-specified model"
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -2.0689926147460938,
"B": -0.799102783203125,
"C": -1.2588729858398438,
"D": -1.9858283996582031
}
},
{
"ds_idx": 13824,
"subject": "virology",
"question": "The requirements for an effective surveillance system include:",
"choices": [
"A diagnostic algorithm",
"Staff members",
"A sampling frame",
"All of the above"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -2.169858932495117,
"B": -2.687652587890625,
"C": -2.289705276489258,
"D": -0.3370285928249359
}
},
{
"ds_idx": 1087,
"subject": "college_computer_science",
"question": "At time 0, five jobs are available for execution on a single processor, with service times of 25, 15, 5, 3, and 2 time units. Which of the following is the minimum value of the average completion time of these jobs?",
"choices": [
"50",
"208/5",
"92/5",
"10"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.5780178308486938,
"B": -1.2319389581680298,
"C": -1.2987152338027954,
"D": -1.5095363855361938
}
},
{
"ds_idx": 6645,
"subject": "jurisprudence",
"question": "Why does the 'capability' approach to justice reject Rawls' theory?",
"choices": [
"Because it fails to address the actual capabilities people have to benefit from his theory of justice.",
"Because it ignores the duties of individuals.",
"Because it applies only to democratic societies.",
"Because it overlooks the importance of the courts."
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -1.1337003707885742,
"B": -1.5565004348754883,
"C": -1.3356943130493164,
"D": -1.6019887924194336
}
},
{
"ds_idx": 6058,
"subject": "high_school_world_history",
"question": "This question refers to the following information.\nOh, shame on you, poor Winter King!\nWhat's this that you have done?\nIs 't not a very naughty thing\nTo snatch the kaiser's crown?\nNow you will have to stay away\nAlike from Rhine and Prague,\nAnd more than that-shame and dismay\nYour days and nights will plague.\nRight well you knew, and all the world,\nRight well they know this thing,\nThat Ferdinand alone can be\nBohemia's lawful king.\nSo come, dear Fritz, rouse up and go\nTo Ferdinand, your king,\nAnd beg him graciously to show\nFull pardon for your sin.\n\"Poor Winter King,\" seventeenth-century song\nWhich of the following best explains why the narrator mentions \"Rhine and Prague\"?",
"choices": [
"They are destinations for the Winter King's planned pilgrimage.",
"These are lands that the Winter King lost in battle.",
"Because these were strongholds of the Holy Roman Empire, the narrator warns the Winter King to stay away.",
"They are significant military posts within the borders of the Holy Roman Empire."
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4815866947174072,
"B": -1.669189691543579,
"C": -0.9281617999076843,
"D": -1.671114206314087
}
},
{
"ds_idx": 2941,
"subject": "high_school_biology",
"question": "The endocrine system maintains homeostasis using many feedback mechanisms. Which of the following is an example of positive feedback?",
"choices": [
"Infant suckling causes a mother's brain to release oxytocin, which in turn stimulates milk production.",
"An enzyme is allosterically inhibited by the product of the reaction it catalyzes.",
"When ATP is abundant the rate of glycolysis decreases.",
"When blood sugar levels decrease to normal after a meal, insulin is no longer secreted."
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.2904571294784546,
"B": -1.3918060064315796,
"C": -1.7378238439559937,
"D": -1.2099250555038452
}
},
{
"ds_idx": 3449,
"subject": "high_school_geography",
"question": "The tendency for migration to decrease with distance is called",
"choices": [
"push factors.",
"pull factors.",
"distance decay.",
"migration selectivity."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.5353158712387085,
"B": -1.3025487661361694,
"C": -1.0775750875473022,
"D": -1.7713521718978882
}
},
{
"ds_idx": 12633,
"subject": "professional_psychology",
"question": "Nonverbal and primarily nonculturally biased abilities are referred to as:",
"choices": [
"Fluid intelligence",
"Broad stratum abilities",
"Achievement or aptitude",
"Crystallized intelligence"
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.426281452178955,
"B": -1.5873799324035645,
"C": -1.2332863807678223,
"D": -1.341261386871338
}
},
{
"ds_idx": 6574,
"subject": "jurisprudence",
"question": "Who was an exponent of \u201cnatural law with a variable content\u201d?",
"choices": [
"John Rawls",
"Stammler",
"Jerome Hall",
"John Finns"
],
"correct_idx": 1,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.1486749649047852,
"B": -1.4046545028686523,
"C": -1.5406064987182617,
"D": -1.5270357131958008
}
},
{
"ds_idx": 2593,
"subject": "global_facts",
"question": "Which of the following is true as of 2020?",
"choices": [
"males are more likely to have depression and are more likely to have an alcohol use disorder than females.",
"females are more likely to have depression and are more likely to have an alcohol use disorder than males.",
"males are more likely than females to have depression while females are more likely than males to have an alcohol use disorder.",
"females are more likely than males to have depression while males are more likely than females to have an alcohol use disorder."
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4254014492034912,
"B": -1.4058263301849365,
"C": -1.2855432033538818,
"D": -1.4550931453704834
}
},
{
"ds_idx": 10338,
"subject": "prehistory",
"question": "Which area is home to the tradition of making stone tools from chipped pebbles, called Hoabinhian?",
"choices": [
"Australia",
"North America",
"Europe",
"Asia"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.4697381258010864,
"B": -1.456426739692688,
"C": -1.3773823976516724,
"D": -1.2681390047073364
}
},
{
"ds_idx": 895,
"subject": "college_biology",
"question": "Which of the following is the symplastic pathway for the movement of sucrose from the site of photosynthesis in mesophyll cells into the phloem?",
"choices": [
"Fibers, phloem parenchyma, companion cell, sieve tube",
"Phloem parenchyma, fibers, bundle sheath, tracheids",
"Companion cells, phloem parenchyma, fibers, sieve tube",
"Bundle sheath, phloem parenchyma, companion cell, sieve tube"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.5421520471572876,
"B": -1.6173816919326782,
"C": -1.0885502099990845,
"D": -1.394797921180725
}
},
{
"ds_idx": 11842,
"subject": "professional_law",
"question": "A gambler learned that a friend of his was planning a vacation to visit some casinos. The gambler, who was an avid college football fan, approached his friend. The gambler explained that he wanted to place a bet with a sports-betting agency that his beloved team would win the college football championship the next year. The gambler further explained that he had read that the odds of his team winning the football championship next year were listed as 100-1, and he wanted to place a $1,000 bet on his team. The gambler told his friend that he would pay him for his efforts. After further discussion, the gambler wrote up the following agreement:\"I hereby promise to pay my friend $100 if he will place a $1,000 bet for me that my favorite team will win the college football championship next year. The bet is to be placed at my lucky casino sports book. I further promise to pay my friend an additional $100 within 30 days after the bet is placed. \"Since the gambler knew his wife would be angry if she found out about the bet, the gambler included a clause that the friend promised not to tell anyone about the bet. After both the gambler and his friend signed the agreement, the gambler gave him $1,100, representing $1,000 to place the bet and $100 as the initial payment for the friend's efforts. The friend then went on vacation. Shortly after arrival, the friend attempted to place the bet for the gambler. However, the friend discovered that the gambler's lucky casino had recently closed its sports book. The friend then entered another casino next door and discovered that they had a sports book, also offering 100-1 odds on the gambler's favorite team winning the college football championship the next year. The friend placed the bet with the other casino's sports book. After returning from his vacation, the friend gave the betting slip to the gambler. Nothing more was said until 30 days later, when the friend asked the gambler when he was going to receive the final $100. The gambler then refused to pay anything more to the friend. In a breach of contract action by the friend to collect the $100, which of the following, if true, is the gambler's best defense?",
"choices": [
"The contract was void ab initio, because the contract could not be performed in its entirety within the span of one year.",
"The friend's placing the $1,000 bet at the other casino instead of at the gambler's lucky casino constituted a material breach of contract, thereby relieving the gambler of any further contractual obligations under the terms of their agreement.",
"While on vacation, the friend told his girlfriend about the $1,000 that he bet on the gambler's behalf.",
"Since the gambler's lucky casino no longer had a sports book at which to place the bet, all contractual obligations would be suspended under the doctrine of frustration of purpose."
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4980014562606812,
"B": -1.4251006841659546,
"C": -1.3503612279891968,
"D": -1.2955058813095093
}
},
{
"ds_idx": 6808,
"subject": "logical_fallacies",
"question": "Which of the following best describes the fallacy of self-evident truths?",
"choices": [
"arguing that a claim is true based only on evidence that an authority figure accepts the claim.",
"arguing a claim should be accepted based only on evidence that unidentified authorities accept it.",
"arguing that a claim should be accepted based on evidence that is not presented, but is asserted to be well known or obvious.",
"arguing someone should accept a claim based only on the evidence that he or she has the favorable characteristics to understand the truth of the claim."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.4917902946472168,
"B": -1.5544219017028809,
"C": -1.1812872886657715,
"D": -1.3760638236999512
}
},
{
"ds_idx": 419,
"subject": "business_ethics",
"question": "Predatory pricing.",
"choices": [
"1,2,4",
"1,2,3,4",
"1,2",
"1,4"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.5335991382598877,
"B": -0.9945347905158997,
"C": -1.6542103290557861,
"D": -1.5247032642364502
}
},
{
"ds_idx": 3459,
"subject": "high_school_geography",
"question": "What is the most rapidly growing religion in the United States today?",
"choices": [
"Islam",
"Christianity",
"Hinduism",
"Buddhism"
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.3357406854629517,
"B": -0.9927936792373657,
"C": -1.565921425819397,
"D": -1.892114281654358
}
},
{
"ds_idx": 13638,
"subject": "us_foreign_policy",
"question": "What is direct diplomacy?",
"choices": [
"Members of Congress negotiating directly with foreign governments",
"Face-to-face meetings between state leaders",
"The president consulting Congress on foreign policy issues",
"Bilateral talks that do not involve a third-party negotiator"
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.7076327800750732,
"B": -1.5830905437469482,
"C": -1.6948649883270264,
"D": -0.8533904552459717
}
},
{
"ds_idx": 11519,
"subject": "professional_law",
"question": "Calvin Poultry Co. offered to sell Chickenshop 20,000 pounds of chicken at 40 cents per pound under specified delivery terms. Chickenshop accepted the offer as follows: \"We accept your offer for 20,000 pounds of chicken at 40 cents per pound per city scale weight certificate.\"\u009d Which of the following is correct?",
"choices": [
"A contract was formed on Calvin's terms.",
"Chickenshop's reply constitutes a conditional acceptance, but not a counteroffer.",
"Chickenshop's reply constitutes a counteroffer and no contract was formed.",
"A contract was formed on Chickenshop's terms."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.5545849800109863,
"B": -1.5988564491271973,
"C": -1.2391552925109863,
"D": -1.228508472442627
}
},
{
"ds_idx": 10309,
"subject": "prehistory",
"question": "Archaeology differs from pseudoscientific speculation and fantasies about ancient extraterrestrials because:",
"choices": [
"archaeology is not confined to evidence or the scientific method.",
"archaeologists are not interested in science fiction and do not actually know much about the ancient past.",
"archaeologists have actually solved all of the mysteries and controversies about the past.",
"archaeologists follow the scientific method and are constrained by evidence or lack of evidence."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.5578113794326782,
"B": -1.6582332849502563,
"C": -1.3698784112930298,
"D": -1.073224663734436
}
},
{
"ds_idx": 2023,
"subject": "electrical_engineering",
"question": "In force-current analogy, electrical analogous quantity for displacement (x) is",
"choices": [
"voltage.",
"inductance.",
"capacitance.",
"flux."
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4585504531860352,
"B": -1.281764030456543,
"C": -1.100520133972168,
"D": -1.8737382888793945
}
},
{
"ds_idx": 9275,
"subject": "moral_scenarios",
"question": "For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | When I took out the trash, I included used paper plates. Scenario 2 | I wanted the diamonds, so I purchased them from the jeweler.",
"choices": [
"Wrong, Wrong",
"Wrong, Not wrong",
"Not wrong, Wrong",
"Not wrong, Not wrong"
],
"correct_idx": 3,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.2316827774047852,
"B": -1.398869514465332,
"C": -1.4156408309936523,
"D": -1.5890569686889648
}
},
{
"ds_idx": 1148,
"subject": "college_mathematics",
"question": "Let S be a compact topological space, let T be a topological space, and let f be a function from S onto T. Of the following conditions on f, which is the weakest condition sufficient to ensure the compactness of T?",
"choices": [
"f is a homeomorphism.",
"f is continuous and one-to-one",
"f is continuous.",
"f is one-to-one"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.4029884338378906,
"B": -1.130645751953125,
"C": -1.5387191772460938,
"D": -1.5725288391113281
}
},
{
"ds_idx": 10010,
"subject": "philosophy",
"question": "According to Butler, the degree to which one acts compassionately depends on:",
"choices": [
"the strength of his affection of compassion.",
"the strength of his other affections besides compassion.",
"both a and b.",
"neither a nor b."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.4090372323989868,
"B": -1.400103211402893,
"C": -1.1051527261734009,
"D": -1.7587629556655884
}
},
{
"ds_idx": 3012,
"subject": "high_school_chemistry",
"question": "The melting point of straight-chain hydrocarbons increases as the number of carbon atoms increase. The reason for this is the",
"choices": [
"increasing mass of the compounds",
"increasing polarity of the compounds",
"increasing number of induced dipoles per molecule",
"increased probability of hydrogen bonds"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.5858407020568848,
"B": -1.3796753883361816,
"C": -1.1909470558166504,
"D": -1.4395928382873535
}
},
{
"ds_idx": 7191,
"subject": "marketing",
"question": "Salespeople who view a situation as a customer sees it are exhibiting",
"choices": [
"cooperation",
"empathy",
"responsibility",
"tolerance"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.4805947542190552,
"B": -1.0158520936965942,
"C": -1.4016228914260864,
"D": -1.8194009065628052
}
},
{
"ds_idx": 1526,
"subject": "computer_security",
"question": "What are the types of scanning?",
"choices": [
"Port, network, and services",
"Network, vulnerability, and port ",
"Passive, active, and interactive",
"Server, client, and network"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.5466312170028687,
"B": -1.303737998008728,
"C": -1.1738723516464233,
"D": -1.5957950353622437
}
},
{
"ds_idx": 13106,
"subject": "public_relations",
"question": "The ________ phase of the PR process consists of making decisions about which strategies and tactics will be used to respond to the goals of the organization and executing those strategies and tactics.",
"choices": [
"planning",
"implementation",
"evaluation",
"tactical"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.3785674571990967,
"B": -0.8718211054801941,
"C": -1.5841128826141357,
"D": -2.0861709117889404
}
},
{
"ds_idx": 174,
"subject": "anatomy",
"question": "Examination of a patient indicates that they have a medially directed strabismus (squint). This could be due to damage to the",
"choices": [
"oculomotor nerve.",
"trochlear nerve.",
"ophthalmic trigeminal nerve.",
"abducens nerve."
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.3101080656051636,
"B": -1.230829119682312,
"C": -1.6417158842086792,
"D": -1.4215372800827026
}
},
{
"ds_idx": 10185,
"subject": "prehistory",
"question": "The Egyptian system of hieroglyphics:",
"choices": [
"did not use pictographs",
"appears to have developed suddenly",
"was the earliest form of writing in the world",
"all of the above"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.956963062286377,
"B": -2.2050042152404785,
"C": -1.8462929725646973,
"D": -0.5320019125938416
}
},
{
"ds_idx": 1456,
"subject": "college_physics",
"question": "Unpolarized light is incident on a pair of ideal linear polarizers whose transmission axes make an angle of 45q with each other. The transmitted light intensity through both polarizers is what percentage of the incident intensity?",
"choices": [
"100%",
"75%",
"50%",
"25%"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4990825653076172,
"B": -1.522817611694336,
"C": -1.202096939086914,
"D": -1.4230880737304688
}
},
{
"ds_idx": 938,
"subject": "college_chemistry",
"question": "The strongest base in liquid ammonia is",
"choices": [
"NH3",
"NH2\u2212",
"NH4+",
"N2H4"
],
"correct_idx": 1,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.0545523166656494,
"B": -1.453615427017212,
"C": -1.549541711807251,
"D": -1.603485345840454
}
},
{
"ds_idx": 7463,
"subject": "miscellaneous",
"question": "Old-fashioned steam engines were powered by coal transferred from the tender to the engine by human stokers using a shovel. What is the maximum power such an engine could attain?",
"choices": [
"6*10^3 W",
"6*10^5 W",
"6*10^7 W",
"6*10^11 W"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4481947422027588,
"B": -1.4223825931549072,
"C": -1.2777941226959229,
"D": -1.4315760135650635
}
},
{
"ds_idx": 4162,
"subject": "high_school_macroeconomics",
"question": "An increase in corporate optimism will have which of the following effects in the market for loanable funds?",
"choices": [
"An increase in supply lowering the interest rate.",
"A decrease in demand increasing the interest rate.",
"An increase in both supply and demand and an ambiguous change in interest rates.",
"An increase in demand increasing the interest rate."
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.5666868686676025,
"B": -1.5011694431304932,
"C": -1.0390284061431885,
"D": -1.5604708194732666
}
},
{
"ds_idx": 333,
"subject": "astronomy",
"question": "What is true about the rotation of a solid disk?",
"choices": [
"The velocity is the same on every point on the disk",
"The part closer to the axis has a longer period",
"The period of rotation is the same on every point on the disk",
"The part closer to the axis has a shorter period"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.656703233718872,
"B": -1.796891450881958,
"C": -0.777093231678009,
"D": -1.7268002033233643
}
},
{
"ds_idx": 6841,
"subject": "machine_learning",
"question": "Let us say that we have computed the gradient of our cost function and stored it in a vector g. What is the cost of one gradient descent update given the gradient?",
"choices": [
"O(D)",
"O(N)",
"O(ND)",
"O(ND^2)"
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.7901649475097656,
"B": -1.0815982818603516,
"C": -1.3981971740722656,
"D": -1.4172477722167969
}
},
{
"ds_idx": 5074,
"subject": "high_school_psychology",
"question": "People who have previously not gotten along are often able to put aside their differences in the face of a mutual crisis. Their need to work together is often referred to as a(n)",
"choices": [
"communal goal",
"superordinate goal",
"subordinate goal",
"alliance formation"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.542922019958496,
"B": -0.7254839539527893,
"C": -2.1646604537963867,
"D": -1.683455467224121
}
},
{
"ds_idx": 11359,
"subject": "professional_law",
"question": "An investor owned substantial oil and mineral interests in the Rocky Mountains. In July 1990, the investor acquired title to a 100-acre tract located just outside the city limits in a rural area. In 1997, the investor decided to sell his vast ranching and oil interests in the Rocky Mountains. In July, the investor conveyed the tract for a consideration of one dollar, receipt of which was acknowledged, \"to my nephew, his heirs, and assigns, but if my nephew's wife dies without issue, to my chef and her heirs and assigns. \"After taking possession of the 100-acre tract, the nephew discovered considerable oil reserves on the property. He then began oil drilling operations and removed large quantities of oil, which he sold. At no time did he give notice to the chef of his oil-drilling activities. Months passed before the chef learned that the nephew was selling oil from the drilling operations. The chef has now brought an action in equity for an accounting of the value of the oil removed and for an injunction against any further oil removal. If the nephew's wife died without issue before the beginning of the nephew's oil drilling operations, would this improve the likelihood of the chef's success than if the wife were still alive?",
"choices": [
"No, because the right to remove oil is an incident of the right to possession.",
"No, because the nature of the nephew's estate would not be altered by the wife's death.",
"Yes, because the wife's death without issue would convert the nephew's fee into a reversionary interest.",
"Yes, because although the nephew still has a fee, it would now be certain to terminate."
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.8841612339019775,
"B": -1.7150328159332275,
"C": -0.7763044834136963,
"D": -1.5979654788970947
}
},
{
"ds_idx": 5930,
"subject": "high_school_world_history",
"question": "This question refers to the following information.\n\"The spontaneous forces of capitalism have been steadily growing in the countryside in recent years, with new rich peasants springing up everywhere and many well-to-do middle peasants striving to become rich peasants. On the other hand, many poor peasants are still living in poverty for lack of sufficient means of production, with some in debt and others selling or renting out their land. If this tendency goes unchecked, the polarization in the countryside will inevitably be aggravated day by day. Those peasants who lose their land and those who remain in poverty will complain that we are doing nothing to save them from ruin or to help them overcome their difficulties. Nor will the well-to-do middle peasants who are heading in the capitalist direction be pleased with us, for we shall never be able to satisfy their demands unless we intend to take the capitalist road. Can the worker-peasant alliance continue to stand in these circumstances? Obviously not! There is no solution to this problem except on a new basis. And that means to bring about, step by step, the socialist transformation of the whole of agriculture simultaneously with the gradual realization of socialist industrialization and the socialist transformation of handicrafts and capitalist industry and commerce; in other words, it means to carry out co-operation and eliminate the rich-peasant economy and the individual economy in the countryside so that all the rural people will become increasingly well off together. We maintain that this is the only way to consolidate the worker-peasant alliance.\"\nMao Zedong, On the Question of Agricultural Co-operation, 1955\nMao's view of the cooperation of peasant labor most directly reflects the influence of which of the following?",
"choices": [
"The ideals of communism as stated by Joseph Stalin",
"The ideals of the Enlightenment as stated by political revolutionaries such as Maximilian Robespierre",
"The ideals of classical liberalism as stated by Adam Smith",
"The ideals of globalization as evidenced by multinational trade blocs such as NAFTA or the European Union"
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.3369324207305908,
"B": -1.4051353931427002,
"C": -1.1527550220489502,
"D": -1.7491428852081299
}
},
{
"ds_idx": 2604,
"subject": "global_facts",
"question": "As of 2019, about what percentage of Turks say God plays an important role in their life?",
"choices": [
"59%",
"69%",
"79%",
"89%"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.6824238300323486,
"B": -1.4776794910430908,
"C": -1.1438095569610596,
"D": -1.339514970779419
}
},
{
"ds_idx": 9698,
"subject": "nutrition",
"question": "Which vitamin is required for vision in dim light?\n",
"choices": [
"Vitamin A",
"Vitamin D",
"Vitamin E",
"Vitamin K"
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -1.0516940355300903,
"B": -1.3899909257888794,
"C": -1.72083580493927,
"D": -1.5191336870193481
}
},
{
"ds_idx": 965,
"subject": "college_chemistry",
"question": "What is the limiting high-temperature molar heat capacity at constant volume (C_V) of a gas-phase diatomic molecule?",
"choices": [
"1.5R",
"2R",
"2.5R",
"3.5R"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.421993374824524,
"B": -1.2556191682815552,
"C": -1.4602395296096802,
"D": -1.4332925081253052
}
},
{
"ds_idx": 9673,
"subject": "nutrition",
"question": "Deficiency of vitamin A in children causes:\n",
"choices": [
"Goitre",
"Poor cognitive development",
"Poor bone growth",
"Increased risk of mortality"
],
"correct_idx": 3,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.1514606475830078,
"B": -1.4595680236816406,
"C": -1.2322254180908203,
"D": -1.8481731414794922
}
},
{
"ds_idx": 6130,
"subject": "human_aging",
"question": "Which of the following could very well be a cohort difference rather than a change that occurs with age?",
"choices": [
"Differences in weight",
"Shrinkage of the thymus gland",
"Enlargement of the heart",
"Increasing size of nose and ears"
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.4339832067489624,
"B": -1.2813552618026733,
"C": -1.3343032598495483,
"D": -1.5287994146347046
}
},
{
"ds_idx": 7364,
"subject": "medical_genetics",
"question": "Which of the following diagnostic techniques is of no value for the diagnosis of neural tube defects?",
"choices": [
"Amniocentesis",
"Chorion villus sampling (CVS)",
"Maternal serum screening",
"Ultrasonography"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.3702471256256104,
"B": -1.3674814701080322,
"C": -1.3909876346588135,
"D": -1.4228003025054932
}
},
{
"ds_idx": 9741,
"subject": "philosophy",
"question": "According to Socrates, an unexamined life is not worth living; and it certainly could not be a virtuous life. Why not?",
"choices": [
"Because if someone did not know how to act virtuously, he or she would still be considered virtuous by others who also did not know the principles for good living.",
"Because since Socrates was a philosopher, he of course thought that people who examined their lives philosophically were more virtuous than those who did not.",
"Because without knowing the rationale for why one should act in a particular way, one does not know whether actions are justified and ought to be repeated.",
"Because a virtuous life would be one in which someone does what the rest of the society says is right, and that means examining views other than one's own."
],
"correct_idx": 2,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.3072030544281006,
"B": -1.314288854598999,
"C": -1.5680139064788818,
"D": -1.4199655055999756
}
},
{
"ds_idx": 451,
"subject": "business_ethics",
"question": "Managers are entrusted to run the company in the best interest of ________. Specifically, they have a duty to act for the benefit of the company, as well as a duty of ________ and of _______.",
"choices": [
"Shareholders, Care and Skill, Diligence",
"Stakeholder, Care and Skill, Diligence",
"Shareholders, Self-interest, Diligence",
"Stakeholders, Care and skill, Self-interest"
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.9083900451660156,
"B": -0.8034800887107849,
"C": -1.6768569946289062,
"D": -1.5491008758544922
}
},
{
"ds_idx": 5496,
"subject": "high_school_statistics",
"question": "There are two games involving flipping a fair coin. In the first game you win a prize if you can throw between 45% and 55% heads. In the second game you win if you can throw more than 80% heads. For each game would you rather flip the coin 30 times or 300 times?",
"choices": [
"30 times for each game",
"300 times for each game",
"30 times for the first game and 300 times for the second",
"300 times for the first game and 30 times for the second"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.6659162044525146,
"B": -1.3767964839935303,
"C": -1.2698171138763428,
"D": -1.3084924221038818
}
},
{
"ds_idx": 7607,
"subject": "miscellaneous",
"question": "In what city is TV's 'The Mary Tyler Moore Show' set?",
"choices": [
"Dallas",
"Chicago",
"New York",
"Minneapolis"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.3732339143753052,
"B": -1.2089482545852661,
"C": -1.372198224067688,
"D": -1.6619149446487427
}
},
{
"ds_idx": 3226,
"subject": "high_school_computer_science",
"question": "Which of the following is correct about Python?",
"choices": [
"It supports automatic garbage collection.",
"It can be easily integrated with C, C++, COM, ActiveX, CORBA, and Java.",
"Both of the above.",
"None of the above."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.7183901071548462,
"B": -1.514971375465393,
"C": -0.8579527139663696,
"D": -1.7683607339859009
}
},
{
"ds_idx": 5287,
"subject": "high_school_psychology",
"question": "A pencil and paper personality test that places a person in one of several personality categories (such as extrovert/introvert, etc.) is based on which personality theory?",
"choices": [
"psychodynamic",
"trait",
"biological",
"behaviorist"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.3821759223937988,
"B": -1.1022038459777832,
"C": -1.9560322761535645,
"D": -1.298536777496338
}
},
{
"ds_idx": 6101,
"subject": "human_aging",
"question": "Which of the following is most strongly associated with loss of bone density?",
"choices": [
"Genetic factors",
"Lifestyle factors like not exercising",
"Hormonal factors such as loss of estrogen",
"Cohort factors"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.408810019493103,
"B": -1.5614665746688843,
"C": -1.1284927129745483,
"D": -1.51581609249115
}
},
{
"ds_idx": 10537,
"subject": "professional_accounting",
"question": "Mentor Co. a U.S. corporation owned 100% of a Swiss corporation. The Swiss franc is the functional currency. The remeasurement of Mentor's financial statements resulted in a $25000 gain at year end. The translation of the financial statements resulted in a $40000 gain at year end. What amount should Mentor recognize as foreign currency gain in its income statement?",
"choices": [
"$0",
"$25,000",
"$40,000",
"65000"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -2.05108380317688,
"B": -1.0972282886505127,
"C": -1.3467724323272705,
"D": -1.311255693435669
}
},
{
"ds_idx": 1800,
"subject": "conceptual_physics",
"question": "The magnetic field lines about a current-carrying wire form",
"choices": [
"circles",
"radial lines",
"eddy currents",
"energy loops"
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.2734802961349487,
"B": -1.2241371870040894,
"C": -1.6079071760177612,
"D": -1.5141342878341675
}
},
{
"ds_idx": 6937,
"subject": "machine_learning",
"question": "You are training a linear regression model for a simple estimation task, and notice that the model is overfitting to the data. You decide to add in $\\ell_2$ regularization to penalize the weights. As you increase the $\\ell_2$ regularization coefficient, what will happen to the bias and variance of the model?",
"choices": [
"Bias increase ; Variance increase",
"Bias increase ; Variance decrease",
"Bias decrease ; Variance increase",
"Bias decrease ; Variance decrease"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.3435629606246948,
"B": -1.4473799467086792,
"C": -1.3401297330856323,
"D": -1.433099627494812
}
},
{
"ds_idx": 1106,
"subject": "college_mathematics",
"question": "A discrete graph is complete if there is an edge connecting any pair of vertices. How many edges does a complete graph with 10 vertices have?",
"choices": [
"10",
"20",
"25",
"45"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.3326611518859863,
"B": -1.377565860748291,
"C": -1.3163495063781738,
"D": -1.6058392524719238
}
},
{
"ds_idx": 3835,
"subject": "high_school_government_and_politics",
"question": "Which of the following is NOT part of the \"iron triangles\" that influence public policy-making?",
"choices": [
"House committees",
"Senate committees",
"The courts",
"Bureaucratic agencies"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.6616551876068115,
"B": -1.4294469356536865,
"C": -1.1713616847991943,
"D": -1.3644959926605225
}
},
{
"ds_idx": 13071,
"subject": "public_relations",
"question": "As the single-person public relations staff of a public transit agency, you are tasked with increasing the number of people who ride your buses each month. Your target audiences are lower-income individuals, college students, people with basic educations and people for whom English is a Second Language. When preparing to craft a primary message, what should you do first?",
"choices": [
"Consider the comprehension level of all of the target audiences.",
"Consider how riding the bus could similarly affect all of the target audiences.",
"Consider which mediums would most effectively reach all of the target audiences.",
"Consider the geographic locations where all of the target audiences live and work."
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.499218463897705,
"B": -1.5883755683898926,
"C": -0.9040035605430603,
"D": -1.7967400550842285
}
},
{
"ds_idx": 558,
"subject": "clinical_knowledge",
"question": "During CPR, chest compressions should be delivered at a rate of:",
"choices": [
"80/minute.",
"as fast as possible.",
"100/minute.",
"varies with each patient."
],
"correct_idx": 2,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.255732536315918,
"B": -1.2824487686157227,
"C": -1.4645051956176758,
"D": -1.588393211364746
}
},
{
"ds_idx": 1421,
"subject": "college_physics",
"question": "The eigenvalues of a Hermitian operator are always",
"choices": [
"real",
"imaginary",
"degenerate",
"linear"
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.319764256477356,
"B": -1.551005482673645,
"C": -1.1158820390701294,
"D": -1.6871176958084106
}
},
{
"ds_idx": 1949,
"subject": "electrical_engineering",
"question": "A Binary number system has how many digits.",
"choices": [
"0",
"1",
"2",
"10"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.5701743364334106,
"B": -1.2799960374832153,
"C": -1.3384848833084106,
"D": -1.4035464525222778
}
},
{
"ds_idx": 4080,
"subject": "high_school_macroeconomics",
"question": "Suppose a nation is experiencing an annual budget surplus and uses some of this surplus to pay down part of the national debt. One potential side effect of this policy would be",
"choices": [
"increase interest rates and throw the economy into a recession.",
"increase interest rates and depreciate the nation's currency.",
"decrease interest rates and risk an inflationary period.",
"decrease interest rates and throw the economy into a recession."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.4475321769714355,
"B": -1.5403037071228027,
"C": -1.210404872894287,
"D": -1.3983283042907715
}
},
{
"ds_idx": 13525,
"subject": "sociology",
"question": "The theory of development which suggests that societies move from traditional to modern, industrial forms of organization is called:",
"choices": [
"Westernization theory",
"Modernization theory",
"Industrialization theory",
"Dependency theory"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.9566025733947754,
"B": -0.38786938786506653,
"C": -2.4748387336730957,
"D": -2.366283893585205
}
},
{
"ds_idx": 6819,
"subject": "logical_fallacies",
"question": "Which of the following best describes the fallacy of poisoning the well?",
"choices": [
"a way of suppressing evidence that eliminates a source of evidence from consideration by claiming the source is flawed, when there is no true relationship between the alleged flaw and the reliability of the source",
"arguing that a claim should be accepted based on evidence that is not presented, but asserted to be well known or obvious.",
"arguing something should be done based only on evidence that it's been done that way in the past, or should be believed only because it's been believed for a long time.",
"reasoning that a claim is true based only on the evidence that it has been made so often."
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -1.3520647287368774,
"B": -1.3627229928970337,
"C": -1.4603677988052368,
"D": -1.392717957496643
}
},
{
"ds_idx": 4870,
"subject": "high_school_physics",
"question": "What happens to the pressure, P, of an ideal gas if the temperature is increased by a factor of 2 and the volume is increased by a factor of 8 ?",
"choices": [
"P decreases by a factor of 16.",
"P decreases by a factor of 4.",
"P decreases by a factor of 2.",
"P increases by a factor of 4."
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.3857778310775757,
"B": -1.2370198965072632,
"C": -1.4093316793441772,
"D": -1.609071135520935
}
},
{
"ds_idx": 1359,
"subject": "college_medicine",
"question": "With an increasing number of sprints the:",
"choices": [
"anaerobic contribution progressively increases.",
"pH of the muscle falls below 6.0.",
"blood glucose concentration falls below 3 mmol/L.",
"relative contribution of aerobic metabolism increases."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.3196300268173218,
"B": -1.7653907537460327,
"C": -1.5763648748397827,
"D": -1.0486091375350952
}
},
{
"ds_idx": 5470,
"subject": "high_school_statistics",
"question": "Which of the following is the primary difference between an experiment and an observational study?",
"choices": [
"Experiments are only conducted on human subjects; observational studies can be conducted on nonhuman subjects.",
"In an experiment, the researcher manipulates some variable to observe its effect on a response variable; in an observational study, he or she simply observes and records the observations.",
"Experiments must use randomized treatment and control groups; observational studies also use treatment and control groups, but they do not need to be randomized.",
"Experiments must be double-blind; observational studies do not need to be."
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.4485666751861572,
"B": -1.2423937320709229,
"C": -1.2927477359771729,
"D": -1.6144506931304932
}
},
{
"ds_idx": 6997,
"subject": "management",
"question": "What is a strategic alliance?",
"choices": [
"Any form of partnership between one firm and another",
"Formal agreement committing two or more firms to exchange resources to produce products or services",
"Formal agreement to share profits from a shared investment",
"Formal agreement to share knowledge"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -2.090240716934204,
"B": -0.5097811222076416,
"C": -1.8388655185699463,
"D": -2.1677095890045166
}
},
{
"ds_idx": 13939,
"subject": "world_religions",
"question": "According to the Sikh tradition, the Mul Mantar illuminates the nature of what?",
"choices": [
"Ultimate Reality",
"The gurus",
"The community",
"Sikh worship"
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -0.8219048380851746,
"B": -1.5724999904632568,
"C": -1.6155736446380615,
"D": -1.890186071395874
}
},
{
"ds_idx": 12430,
"subject": "professional_medicine",
"question": "A 25-year-old woman comes to the office because of a 6-month history of increasingly severe low back pain and heavy menses. Her temperature is 37.1\u00b0C (98.8\u00b0F), pulse is 75/min, respirations are 13/min, and blood pressure is 115/79 mm Hg. Physical examination shows no abnormalities. An endometrial biopsy specimen shows regular tubular endometrial glands with abundant mitotic figures in the endometrial glands and stroma. Which of the following proteins or enzymes regulate the progression of cells into this phase of this patient's menstrual cycle?",
"choices": [
"AMP-dependent kinases",
"Cyclin-dependent kinases",
"Hexokinases",
"Lipid kinases"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.924630880355835,
"B": -0.4670121669769287,
"C": -1.9364526271820068,
"D": -2.5012643337249756
}
},
{
"ds_idx": 8414,
"subject": "moral_disputes",
"question": " Ross proposed that we have a duty of reparation-that is, the prima facie duty-to",
"choices": [
"make amends to others for any past wrongs one has done to them.",
"show gratitude toward one's benefactors.",
"refrain from harming others.",
"improve oneself with respect to one's own virtue and knowledge."
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4524743556976318,
"B": -1.6759469509124756,
"C": -1.2994916439056396,
"D": -1.20231032371521
}
},
{
"ds_idx": 6872,
"subject": "machine_learning",
"question": "Statement 1| The kernel density estimator is equivalent to performing kernel regression with the value Yi = 1/n at each point Xi in the original data set. Statement 2| The depth of a learned decision tree can be larger than the number of training examples used to create the tree.",
"choices": [
"True, True",
"False, False",
"True, False",
"False, True"
],
"correct_idx": 1,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.1998025178909302,
"B": -1.4475938081741333,
"C": -1.5020753145217896,
"D": -1.4457799196243286
}
},
{
"ds_idx": 342,
"subject": "astronomy",
"question": "What would weigh the most on the moon?",
"choices": [
"A kilogram of feathers",
"Five pounds of bricks as measured on Earth",
"Five kilograms of feathers",
"A kilogram of bricks"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.4384920597076416,
"B": -1.3973219394683838,
"C": -1.2312185764312744,
"D": -1.5191729068756104
}
},
{
"ds_idx": 1853,
"subject": "econometrics",
"question": "Which of the following is a DISADVANTAGE of using pure time-series models (relative to structural models)?",
"choices": [
"They are not theoretically motivated",
"They cannot produce forecasts easily",
"They cannot be used for very high frequency data",
"It is difficult to determine the appropriate explanatory variables for use in pure time-series models"
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5720298290252686,
"B": -1.464909315109253,
"C": -1.517449140548706,
"D": -1.0836436748504639
}
},
{
"ds_idx": 298,
"subject": "astronomy",
"question": "Which of the following most likely explains why Venus does not have a strong magnetic field?",
"choices": [
"Its rotation is too slow.",
"It has too thick an atmosphere.",
"It is too large.",
"It does not have a metallic core."
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -1.3013896942138672,
"B": -1.3577842712402344,
"C": -1.491943359375,
"D": -1.4147300720214844
}
},
{
"ds_idx": 8284,
"subject": "moral_disputes",
"question": " In Rachels's thought experiment with Smith and Jones, Smith's action (i.e., his holding the child underwater) is supposed to be an example of",
"choices": [
"killing.",
"letting die.",
"assisted suicide.",
"none of the above"
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.3850862979888916,
"B": -1.5498888492584229,
"C": -1.0160391330718994,
"D": -1.7793371677398682
}
},
{
"ds_idx": 7106,
"subject": "marketing",
"question": " This is a marketing communications activity concerned with providing support for the sales force and merchandising personnel:",
"choices": [
"Store marketing.",
"Field marketing.",
"Sales promotions.",
"Personal selling."
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.5851975679397583,
"B": -1.287185788154602,
"C": -1.4270135164260864,
"D": -1.290981411933899
}
},
{
"ds_idx": 12219,
"subject": "professional_medicine",
"question": "A phase 2, multicenter trial was conducted to determine the efficacy of a new vaccine for prevention of HIV infection. The study enrolled 4000 subjects, aged 20 to 65 years. Of these subjects, 2100 were men and 1900 were women; 2500 were white, 1000 were African American, 300 were Hispanic, and 200 were Asian/Pacific Islanders. Results of the trial showed no overall benefit of the vaccine. However, post hoc analysis disclosed a small but statistically significant vaccine protection among African American subjects. Which of the following is the most accurate rationale for questioning the validity of efficacy analysis of the HIV vaccine among the African American study subjects?",
"choices": [
"HIV infection is more prevalent among African American populations",
"The study was not blinded",
"There was a Type II error",
"Vaccine response among African American subjects was not the primary outcome measure"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.6605825424194336,
"B": -1.7610864639282227,
"C": -1.213444709777832,
"D": -1.078211784362793
}
},
{
"ds_idx": 6729,
"subject": "logical_fallacies",
"question": "Which of the following best describes the fallacy of figure of speech?",
"choices": [
"Using emotionally charged language to create an impression about the subject of a claim, without making an argument that the language fits the subject.",
"Treating something that exists in the mind as if it was a real object.",
"Confusing figurative language with literal language.",
"A form of loaded language that uses an emotionally charged restatement of the claim in place of support for the claim."
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5607560873031616,
"B": -1.5211061239242554,
"C": -1.4646333456039429,
"D": -1.087790846824646
}
},
{
"ds_idx": 66,
"subject": "abstract_algebra",
"question": "Statement 1 | If R is a ring and f(x) and g(x) are in R[x], then deg (f(x)+g(x)) = maximum (deg f(x), deg g(x)). Statement 2 | If F is a field then every ideal of F[x] is principal.",
"choices": [
"True, True",
"False, False",
"True, False",
"False, True"
],
"correct_idx": 3,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.1621770858764648,
"B": -1.3846368789672852,
"C": -1.7702207565307617,
"D": -1.368393898010254
}
},
{
"ds_idx": 1544,
"subject": "computer_security",
"question": "A session symmetric key between two parties is used",
"choices": [
"Only once",
"Twice",
"Multiple times",
"Conditions dependant"
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.2434992790222168,
"B": -1.3466753959655762,
"C": -1.001664638519287,
"D": -2.5053553581237793
}
},
{
"ds_idx": 983,
"subject": "college_chemistry",
"question": "Which of the following is a primary standard for use in standardizing bases?",
"choices": [
"Ammonium hydroxide",
"Sulfuric acid",
"Acetic acid",
"Potassium hydrogen phthalate"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.3585896492004395,
"B": -1.3292756080627441,
"C": -1.332287311553955,
"D": -1.560112476348877
}
},
{
"ds_idx": 6087,
"subject": "high_school_world_history",
"question": "This question refers to the following information.\nThe invaders had brought in wheat and other Eurasian and African grains; peach, pear, orange, and lemon trees; chick-peas, grape vines, melons, onions, radishes, and much more. A Spanish nobleman come to America could require his Indians to furnish his table with the fruits of his ancestors.\n\u2014Alfred Crosby, historian, 1972\nWhat was the impact of this process on Europe?",
"choices": [
"The population increased due to the increased diversity of crops.",
"The population declined as a result of imported disease.",
"The population increased due to New World immigrants.",
"The population declined because of famine caused by the mass export of crops."
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.429976224899292,
"B": -1.7540805339813232,
"C": -1.2935702800750732,
"D": -1.1673495769500732
}
},
{
"ds_idx": 2565,
"subject": "formal_logic",
"question": " Construct a complete truth table for the following argument. Then, using the truth table, determine whether the argument is valid or invalid. If the argument is invalid, choose an option which presents a counterexample. (There may be other counterexamples as well.)\n(G \u2261 H) \u00b7 ~I\n~G \u2228 (~H \u2228 I) / G",
"choices": [
"Valid",
"Invalid. Counterexample when H and I are true and G is false",
"Invalid. Counterexample when H is true and I and G are false",
"Invalid. Counterexample when H, I, and G are false"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.3404183387756348,
"B": -1.5830578804016113,
"C": -1.3150334358215332,
"D": -1.5430912971496582
}
},
{
"ds_idx": 10457,
"subject": "professional_accounting",
"question": "Darv Co. had a current ratio of 3-to-1 and a quick ratio of 1-to-1. Current liabilities were $322000. What was the total amount for inventory and prepaid expenses?",
"choices": [
"$322,000",
"$644,000",
"$966,000",
"1288000"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5275158882141113,
"B": -1.3226399421691895,
"C": -1.428539752960205,
"D": -1.3084321022033691
}
},
{
"ds_idx": 8542,
"subject": "moral_scenarios",
"question": "For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I kissed my married coworker after work. Scenario 2 | As I slept in the other room, my roommate opened a map.",
"choices": [
"Wrong, Wrong",
"Wrong, Not wrong",
"Not wrong, Wrong",
"Not wrong, Not wrong"
],
"correct_idx": 1,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.1324905157089233,
"B": -1.4474414587020874,
"C": -1.4736255407333374,
"D": -1.6053184270858765
}
},
{
"ds_idx": 2953,
"subject": "high_school_biology",
"question": "Viruses are considered an exception to the cell theory because they",
"choices": [
"are not independent organisms",
"have only a few genes",
"move about via their tails",
"have evolved from ancestral protists"
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4931647777557373,
"B": -1.7923743724822998,
"C": -1.6781032085418701,
"D": -0.8677967190742493
}
},
{
"ds_idx": 7350,
"subject": "medical_genetics",
"question": "Which of the following disorders is not suitable for population carrier screening?",
"choices": [
"Cystic fibrosis",
"Oculocutaneous albinism",
"Sickle cell disease",
"Tay-Sachs disease"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5713541507720947,
"B": -1.4190123081207275,
"C": -1.491201639175415,
"D": -1.1305410861968994
}
},
{
"ds_idx": 6169,
"subject": "human_aging",
"question": "The most frequent chronic condition for older adults is",
"choices": [
"Hypertension",
"Arthritis",
"Diabetes",
"Cataracts"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.540309190750122,
"B": -1.3884518146514893,
"C": -1.1385071277618408,
"D": -1.5468437671661377
}
},
{
"ds_idx": 8263,
"subject": "moral_disputes",
"question": " Carruthers argues that, on contractualist grounds, nonhuman animals may be accorded which of the following kinds of moral standing?",
"choices": [
"direct",
"indirect",
"quasi-",
"none of the above"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.4380602836608887,
"B": -0.8904433250427246,
"C": -1.7160983085632324,
"D": -1.8077216148376465
}
},
{
"ds_idx": 13884,
"subject": "world_religions",
"question": "To which tradition do the anti-Han rebel group known as the Celestial Masters belong?",
"choices": [
"Buddhism",
"Daoism",
"Confucianism",
"Shamanism"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.3408125638961792,
"B": -0.729333758354187,
"C": -1.8358591794967651,
"D": -2.3626270294189453
}
},
{
"ds_idx": 13026,
"subject": "professional_psychology",
"question": "The Wonderlic Personnel Test is a:",
"choices": [
"measure of psychomotor skills.",
"measure of clerical skills.",
"brief measure of cognitive ability.",
"brief interest inventory."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.6160677671432495,
"B": -2.1664810180664062,
"C": -0.7490366697311401,
"D": -1.5521715879440308
}
},
{
"ds_idx": 10504,
"subject": "professional_accounting",
"question": "A firm's inventory turnover (IT) is 5 times on a cost of goods sold (COGS) of $800,000. If the IT is improved to 8 times while the COGS remains the same, a substantial amount of funds is released from or additionally invested in inventory. In fact,",
"choices": [
"$160,000 is released.",
"$100,000 is additionally invested.",
"$60,000 is additionally invested.",
"$60,000 is released."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.5768053531646729,
"B": -1.6488440036773682,
"C": -1.2467386722564697,
"D": -1.1819689273834229
}
},
{
"ds_idx": 7174,
"subject": "marketing",
"question": " McDonald's and KFC are good examples of:",
"choices": [
"Distributors.",
"Franchising.",
"Merchant.",
"Retailers."
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5248870849609375,
"B": -1.1982917785644531,
"C": -1.871429443359375,
"D": -1.130859375
}
},
{
"ds_idx": 1412,
"subject": "college_physics",
"question": "A uniform solid disk starts from rest and rolls down an inclined plane without slipping. After some time, what fraction of the disk\u2019s total kinetic energy is rotational kinetic energy?",
"choices": [
"1/4",
"1/3",
"1/2",
"2/3"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4138636589050293,
"B": -1.2679705619812012,
"C": -1.1927123069763184,
"D": -1.8387274742126465
}
},
{
"ds_idx": 2350,
"subject": "elementary_mathematics",
"question": "There are 10 red chips and 5 blue chips in a bag. What are the chances of picking a blue chip if you reach into the bag and pick one without looking?",
"choices": [
"1 out of 5",
"2 out of 15",
"5 out of 10",
"5 out of 15"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.7628265619277954,
"B": -1.417016625404358,
"C": -1.1542679071426392,
"D": -1.3436847925186157
}
},
{
"ds_idx": 692,
"subject": "clinical_knowledge",
"question": "Which of the following is true about involuntary movements in the arm?",
"choices": [
"Alcohol makes the tremor of benign essential tremor worse",
"Hemiballismus is due to a stroke causing paralysis of the distal half of the arm",
"A 'milkmaid' grip is sometimes found in dystonia",
"Writer's cramp is an example of a focal dystonia"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.5464606285095215,
"B": -1.3099703788757324,
"C": -1.3755736351013184,
"D": -1.339179515838623
}
},
{
"ds_idx": 4689,
"subject": "high_school_microeconomics",
"question": "A competitive firm's demand for labor is determined directly by",
"choices": [
"profits",
"the opportunity cost of workers\u2019 time",
"the wage and the average (physical) product of labor",
"the marginal (physical) product of labor and the output price"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.7314770221710205,
"B": -1.6485188007354736,
"C": -1.3278095722198486,
"D": -1.014613389968872
}
},
{
"ds_idx": 8885,
"subject": "moral_scenarios",
"question": "For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I gave the student a good grade because he earned it. Scenario 2 | When the Nazis came to my door, I lied and said I didn't know where Ann Frank was.",
"choices": [
"Wrong, Wrong",
"Wrong, Not wrong",
"Not wrong, Wrong",
"Not wrong, Not wrong"
],
"correct_idx": 3,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.216726541519165,
"B": -1.4430525302886963,
"C": -1.4116270542144775,
"D": -1.5401842594146729
}
},
{
"ds_idx": 9125,
"subject": "moral_scenarios",
"question": "For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I hid the love note in the teenager's locker. Scenario 2 | I gripped the woman's hand because she was falling.",
"choices": [
"Wrong, Wrong",
"Wrong, Not wrong",
"Not wrong, Wrong",
"Not wrong, Not wrong"
],
"correct_idx": 3,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.112032175064087,
"B": -1.4080049991607666,
"C": -1.43709397315979,
"D": -1.7305147647857666
}
},
{
"ds_idx": 3222,
"subject": "high_school_computer_science",
"question": "A certain computer game is played between a human player and a computer-controlled player. Every time the computer-controlled player has a turn, the game runs slowly because the computer evaluates all potential moves and selects the best one. Which of the following best describes the possibility of improving the running speed of the game?",
"choices": [
"The game's running speed can only be improved if the game is played between two human players instead of with the computer-controlled player.",
"The game's running speed might be improved by using a process that finds approximate solutions every time the computer-controlled player has a turn.",
"The game's running speed cannot be improved because computers can only be programmed to find the best possible solution.",
"The game's running speed cannot be improved because the game is an example of an algorithm that does not run in a reasonable time."
],
"correct_idx": 1,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.3335750102996826,
"B": -1.4852397441864014,
"C": -1.3856055736541748,
"D": -1.359663724899292
}
},
{
"ds_idx": 6527,
"subject": "international_law",
"question": "What is an act jure gestionis?",
"choices": [
"Acts jure gestionis are those undertaken by States in a private capacity",
"All purchases by the State are acts jure gestionis",
"All acts undertaken by State corporations are jure gestionis",
"All conduct undertaken by government officials in their free time is considered jus gestionis."
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -0.9996407628059387,
"B": -1.5445396900177002,
"C": -1.3862087726593018,
"D": -1.8223755359649658
}
},
{
"ds_idx": 1433,
"subject": "college_physics",
"question": "Two simple pendulums A and B consist of identical masses suspended from strings of length L_A and L_B, respectively. The two pendulums oscillate in equal gravitational fields. If the period of pendulum B is twice the period of pendulum A, which of the following is true of the lengths of the two pendulums?",
"choices": [
"L_B = 4L_A",
"L_B = 2L_A",
"L_B = L_A",
"L_B = 1/2 L_A"
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -1.2880221605300903,
"B": -1.425191044807434,
"C": -1.4374209642410278,
"D": -1.4493743181228638
}
},
{
"ds_idx": 1317,
"subject": "college_medicine",
"question": "All of the following are true regarding the function of neurons EXCEPT:",
"choices": [
"Hyperpolarization at the end of an action potential is one mechanism by which neurons limit the rate at which action potentials may fire.",
"The flow of sodium into the neuron depolarizes the membrane in the first phase of an action potential.",
"The transmitting neuron secretes neurotransmitters into the synaptic cleft from its dendrites.",
"An action potential is initiated when the axon hillock reaches the threshold potential."
],
"correct_idx": 2,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.2788625955581665,
"B": -1.4319311380386353,
"C": -1.4638391733169556,
"D": -1.3922487497329712
}
},
{
"ds_idx": 6594,
"subject": "jurisprudence",
"question": "Which of the following best describes Dworkin's distinction between principles and policies?",
"choices": [
"Principles are legislative, policies are precedents.",
"Principles describe rights, policies describe duties.",
"Principles are democratic, policies are autocratic.",
"Principles describe rights, policies describe goals."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.6141635179519653,
"B": -1.4399806261062622,
"C": -1.3354731798171997,
"D": -1.2085810899734497
}
},
{
"ds_idx": 1232,
"subject": "college_medicine",
"question": "An individual is born with a mutation causing her to partially retain a form of fetal hemoglobin into adulthood. Compared to a normal individual, this person would exhibit:",
"choices": [
"no differences from a normal adult.",
"significantly reduced oxygen binding in the lungs.",
"no symptoms, since retention of fetal hemoglobin would be fatal.",
"increased oxygen binding to hemoglobin in the tissues."
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.5992966890335083,
"B": -1.546657681465149,
"C": -1.1650525331497192,
"D": -1.308773159980774
}
},
{
"ds_idx": 6335,
"subject": "human_sexuality",
"question": "If a woman or man repeatedly experiences pain during intercourse, she will probably want to have intercourse infrequently or not all. In this situation, __________ would likely classify the pain as a punishment.",
"choices": [
"Sigmund Freud",
"John Money",
"B. F. Skinner",
"Havelock Ellis"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.5255157947540283,
"B": -1.0790913105010986,
"C": -1.4784119129180908,
"D": -1.5464699268341064
}
},
{
"ds_idx": 3865,
"subject": "high_school_macroeconomics",
"question": "Required reserves",
"choices": [
"can be used by banks to make loans or buy investments.",
"can be held in a bank's vault or its account at the FED.",
"must be kept in a bank's vault.",
"must be used to make loans."
],
"correct_idx": 1,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.2679725885391235,
"B": -1.3008190393447876,
"C": -1.5021320581436157,
"D": -1.5105091333389282
}
},
{
"ds_idx": 6376,
"subject": "human_sexuality",
"question": "At about _________ months after conception, the gender of the fetus is clear from the appearance of external genitals.",
"choices": [
"2",
"4",
"6",
"8"
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.503149151802063,
"B": -1.3343316316604614,
"C": -1.2934552431106567,
"D": -1.4325124025344849
}
},
{
"ds_idx": 5299,
"subject": "high_school_psychology",
"question": "Which term describes the phenomenon of like-minded groups taking more extreme positions than the individuals that comprise the group?",
"choices": [
"social facilitation",
"deindividuation",
"groupthink",
"group polarization"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.7006832361221313,
"B": -1.7964779138565063,
"C": -1.2784878015518188,
"D": -0.991550087928772
}
},
{
"ds_idx": 4771,
"subject": "high_school_physics",
"question": "If L, M, and T denote the dimensions of length, mass, and time, respectively, what are the dimensions of impulse?",
"choices": [
"LM/T^3",
"LM/T^2",
"LM/T",
"L^2M/T^2"
],
"correct_idx": 2,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.3197600841522217,
"B": -1.3898417949676514,
"C": -1.5434119701385498,
"D": -1.3553168773651123
}
},
{
"ds_idx": 790,
"subject": "college_biology",
"question": "The sight organs of crustaceans and insects contain ommatidia, which make up the individual visual units of the",
"choices": [
"eyespot",
"simple eye",
"compound eye",
"binocular eye"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.9206483364105225,
"B": -1.9581811428070068,
"C": -0.5527684688568115,
"D": -2.0117204189300537
}
},
{
"ds_idx": 2952,
"subject": "high_school_biology",
"question": "Monkeys of South and Central America have prehensile tails, meaning that their tails can be used to grasp objects. The tails of African and Asian monkeys are not prehensile. Which discipline is most likely to provide an evolutionary explanation for how this difference in tails came about?",
"choices": [
"Aerodynamics",
"Biogeography",
"Physiology",
"Biochemistry"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.742621660232544,
"B": -0.5385810732841492,
"C": -1.849524736404419,
"D": -2.497035264968872
}
},
{
"ds_idx": 1349,
"subject": "college_medicine",
"question": "How can several different proteins be produced from the same gene?",
"choices": [
"By selective transcription of different parts of the DNA in the gene.",
"By doubling or tripling the length of the mRNA molecule produced from the primary transcript.",
"The primary transcripts of many genes can be spliced in various ways to produce different mRNAs, a process known as alternative RNA splicing.",
"By splicing of the mRNA molecule after it has been formed from the primary transcript."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.5230350494384766,
"B": -1.7564563751220703,
"C": -0.781211793422699,
"D": -1.9139728546142578
}
},
{
"ds_idx": 6739,
"subject": "logical_fallacies",
"question": "Which of the following fallacies happens when someone argues that a choice must be made among a certain set of alternatives, when there are really other alternatives to choose from?",
"choices": [
"Laudatory personality",
"Accident",
"False dilemma",
"Circular reasoning"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -2.20293927192688,
"B": -1.9937107563018799,
"C": -0.5065585970878601,
"D": -1.9343464374542236
}
},
{
"ds_idx": 6296,
"subject": "human_aging",
"question": "The paradox of well-being refers to the finding of high levels of happiness among",
"choices": [
"Unhappy people",
"Older adults",
"People with very low SES",
"Terminally ill people"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -2.030705690383911,
"B": -0.6888440251350403,
"C": -1.5468723773956299,
"D": -1.8898670673370361
}
},
{
"ds_idx": 1867,
"subject": "econometrics",
"question": "Which of the following is a typical characteristic of financial asset return time-series?",
"choices": [
"Their distributions are thin-tailed",
"They are not weakly stationary",
"They are highly autocorrelated",
"They have no trend"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.3459893465042114,
"B": -1.4765511751174927,
"C": -1.1602765321731567,
"D": -1.6382085084915161
}
},
{
"ds_idx": 9703,
"subject": "nutrition",
"question": "What should elite athletes ideally consume during prolonged high intensity exercise (>2.5 hours)?\n",
"choices": [
"60 g glucose per hour",
"60 g glucose plus fructose per hour",
"90 g glucose per hour",
"90 g glucose plus fructose per hour"
],
"correct_idx": 3,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.2827129364013672,
"B": -1.4157733917236328,
"C": -1.552358627319336,
"D": -1.3290843963623047
}
},
{
"ds_idx": 3649,
"subject": "high_school_government_and_politics",
"question": "The primary function of political action committees (PACs) is to",
"choices": [
"contribute money to candidates for election",
"coordinate local get-out-the-vote campaigns",
"promote the defeat of incumbents in the federal and state legislatures",
"organize protest demonstrations and other acts of civil disobedience"
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.694959044456482,
"B": -1.9699167013168335,
"C": -0.9635270833969116,
"D": -1.2305101156234741
}
},
{
"ds_idx": 5785,
"subject": "high_school_us_history",
"question": "This question refers to the following information.\n\"The Opposition tells us that we ought not to govern a people without their consent. I answer, the rule of liberty that all just government derives its authority from the consent of the governed, applies only to those who are capable of self-government. We govern the Indians without their consent, we govern our territories without their consent, we govern our children without their consent. How do they know that our government would be without their consent? Would not the people of the Philippines prefer the just, human, civilizing government of this Republic to the savage, bloody rule of pillage and extortion from which we have rescued them? And, regardless of this formula of words made only for enlightened, self-governing people, do we owe no duty to the world? Shall we turn these peoples back to the reeking hands from which we have taken them? Shall we abandon them, with Germany, England, Japan, hungering for them? Shall we save them from those nations, to give them a self-rule of tragedy?\u2026Then, like men and not like children, let us on to our tasks, our mission, and our destiny.\"\nAlbert J. Beveridge, \"The March of the Flag,\" 1898\nThe United States became politically engaged with the Philippines after what military conflict?",
"choices": [
"Mexican-American War",
"World War II",
"Spanish-American War",
"The Boxer Rebellion"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -2.3654515743255615,
"B": -1.749255895614624,
"C": -0.4931504726409912,
"D": -2.136056661605835
}
},
{
"ds_idx": 1760,
"subject": "conceptual_physics",
"question": "At very high speeds, an object appears to an observer at rest to be",
"choices": [
"shorter in the direction of travel.",
"shrunken in all directions.",
"shorter in the direction perpendicular to travel.",
"longer in all directions."
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4378533363342285,
"B": -1.412325382232666,
"C": -1.2506356239318848,
"D": -1.482923984527588
}
},
{
"ds_idx": 3057,
"subject": "high_school_chemistry",
"question": "Which net ionic equation below represents a possible reaction that takes place when a strip of magnesium metal is oxidized by a solution of chromium (III) nitrate?",
"choices": [
"Mg(s) + Cr(NO3)3(aq) \u2192 Mg2+(aq) + Cr3+(aq) + 3NO3-(aq)",
"3Mg(s) + 2Cr3+ \u2192 3Mg2+ + 2Cr(s)",
"Mg(s) + Cr3+ \u2192 Mg2+ + Cr(s)",
"3Mg(s) + 2Cr(NO3)3(aq) \u2192 3Mg2+(aq) + 2Cr(s) + NO3-(aq)"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.2869318723678589,
"B": -1.5362299680709839,
"C": -1.684198260307312,
"D": -1.1727179288864136
}
},
{
"ds_idx": 4622,
"subject": "high_school_microeconomics",
"question": "Consider a profit-maximizing firm in a perfectly competitive market with several sellers and several buyers (i.e., the firm is a \"price taker\" of the goods it sells and a \"price taker\" of the hourly wages it pays its workers). If a technological innovation made by someone in this firm were to significantly raise the firm's marginal physical product (but not that of any other firm's), then this innovation would",
"choices": [
"reduce the firm\u2019s employment level, because fewer workers are now needed",
"raise the workers\u2019 hourly wage as they now contribute more marginal revenue",
"lead the firm to hire more workers but not to raise their wages",
"lead the firm to hire more workers and to pay them higher wages"
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4573140144348145,
"B": -1.7156472206115723,
"C": -1.6316933631896973,
"D": -0.9467396140098572
}
},
{
"ds_idx": 518,
"subject": "clinical_knowledge",
"question": "Which of the following is true of psoriatic arthropathy?",
"choices": [
"The absence of plaques of psoriasis excludes the diagnosis",
"The diagnosis can be confirmed by a blood marker",
"Tophi can sometimes be seen over affected joints",
"Nail pitting may give a clue to the diagnosis"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.3953479528427124,
"B": -1.5802196264266968,
"C": -1.0977100133895874,
"D": -1.5591548681259155
}
},
{
"ds_idx": 2334,
"subject": "elementary_mathematics",
"question": "Order the numbers 6, \u22121, 8, \u22123, \u22124 from least to greatest.",
"choices": [
"6, \u22121, 8, \u22123, \u22124",
"\u22124, \u22123, \u22121, 6, 8",
"\u22121, \u22123, \u22124, 6, 8",
"\u22123, \u22124, 6, \u22121, 8"
],
"correct_idx": 1,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.2081592082977295,
"B": -1.4636752605438232,
"C": -1.5077159404754639,
"D": -1.4244964122772217
}
},
{
"ds_idx": 2489,
"subject": "formal_logic",
"question": "Select the best translation into predicate logic: If Delilah is in New York City, then Juana is in Bogota or Layla is in San Juan.",
"choices": [
"Nd \u2283 (Bj \u2022 Sl)",
"Dn \u2261 (Bx \u2022 Ly)",
"Nd \u2261 (Bj \u2022 Sl)",
"Nd \u2283 (Bj \u2228 Sl)"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.513140082359314,
"B": -1.6337531805038452,
"C": -1.010698676109314,
"D": -1.5816081762313843
}
},
{
"ds_idx": 13146,
"subject": "public_relations",
"question": "According to Russian philosopher Mikhail Bakhtin, in order to create ________, or a well-informed public, all communication about an issue must be ________, or take the form of a dialogue.",
"choices": [
"intelligentsia, democratic",
"democracy, dialogic",
"democracy, logical",
"democracy, bipartisan"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -2.53255033493042,
"B": -0.18392527103424072,
"C": -3.3339571952819824,
"D": -2.9658312797546387
}
},
{
"ds_idx": 5383,
"subject": "high_school_psychology",
"question": "Approximately what percent of the population scores between 70 and 130 on the WISC?",
"choices": [
"34",
"50",
"68",
"95"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.7235773801803589,
"B": -1.3620833158493042,
"C": -1.1260069608688354,
"D": -1.4338129758834839
}
},
{
"ds_idx": 13707,
"subject": "virology",
"question": "Globally, the most deaths are caused by:",
"choices": [
"Respiratory infections",
"Diarrheal diseases",
"Malaria",
"Tuberculosis"
],
"correct_idx": 1,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -0.8071578145027161,
"B": -1.2297041416168213,
"C": -1.8446848392486572,
"D": -2.3036463260650635
}
},
{
"ds_idx": 6079,
"subject": "high_school_world_history",
"question": "This question refers to the following information.\n\"Article 1\nThe Parties undertake, as set forth in the Charter of the United Nations, to settle any international dispute in which they may be involved by peaceful means in such a manner that international peace and security and justice are not endangered, and to refrain in their international relations from the threat or use of force in any manner inconsistent with the purposes of the United Nations.\n\"Article 2\nThe Parties will contribute toward the further development of peaceful and friendly international relations by strengthening their free institutions, by bringing about a better understanding of the principles upon which these institutions are founded, and by promoting conditions of stability and well-being. They will seek to eliminate conflict in their international economic policies and will encourage economic collaboration between any or all of them.\n\"Article 3\nIn order more effectively to achieve the objectives of this Treaty, the Parties, separately and jointly, by means of continuous and effective self-help and mutual aid, will maintain and develop their individual and collective capacity to resist armed attack\u2026\n\"Article 5\nThe Parties agree that an armed attack against one or more of them in Europe or North America shall be considered an attack against them all and consequently they agree that, if such an armed attack occurs, each of them, in exercise of the right of individual or collective self-defence recognised by Article 51 of the Charter of the United Nations, will assist the Party or Parties so attacked by taking forthwith, individually and in concert with the other Parties, such action as it deems necessary, including the use of armed force, to restore and maintain the security of the North Atlantic area.\"\nNorth Atlantic Treaty, April 4, 1949\nThe treaty above is most clearly an example of which of the following?",
"choices": [
"A reduction of barriers to trade across international borders",
"A secret treaty to maintain the balance of power in Europe",
"Reconstruction efforts to rebuild Europe after World War II",
"A response to aggression from outside of Western Europe"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.5966510772705078,
"B": -1.28411865234375,
"C": -1.5168132781982422,
"D": -1.2048358917236328
}
},
{
"ds_idx": 2192,
"subject": "elementary_mathematics",
"question": "Jess scored 18 points during her last basketball game. Each basket she made was worth 2 points. How many baskets did she make?",
"choices": [
"20",
"16",
"9",
"8"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.403173804283142,
"B": -1.2568687200546265,
"C": -1.3036788702011108,
"D": -1.6487144231796265
}
},
{
"ds_idx": 12212,
"subject": "professional_medicine",
"question": "During a study of renal glomeruli, a healthy animal kidney is kept in a vascular bath preparation at a constant afferent arterial pressure of 100 mm Hg. If the efferent arteriole is constricted with a vascular clamp, which of the following Starling forces is most likely to change in the glomeruli?",
"choices": [
"Decreased filtration coefficient (Kf)",
"Decreased hydrostatic pressure",
"Decreased oncotic pressure",
"Increased hydrostatic pressure"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.3430938720703125,
"B": -1.3896923065185547,
"C": -1.6336326599121094,
"D": -1.2316398620605469
}
},
{
"ds_idx": 1135,
"subject": "college_mathematics",
"question": "The maximum number of acute angles in a convex 10-gon in the Euclidean plane is",
"choices": [
"1",
"2",
"3",
"4"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.491129994392395,
"B": -1.245392918586731,
"C": -1.3153964281082153,
"D": -1.5484172105789185
}
},
{
"ds_idx": 3429,
"subject": "high_school_european_history",
"question": "This question refers to the following information.\nI am of opinion that the principal and true profession of the Courtier ought to be that of arms; which I would have him follow actively above all else, and be known among others as bold and strong, and loyal to whomsoever he serves. . . .\nI would have him well built and shapely of limb, and would have him show strength and lightness and suppleness, and know all bodily exercises that befit a man of war; whereof I think the first should be to handle every sort of weapon well on foot and on horse, to understand the advantages of each, and especially to be familiar with those weapons that are ordinarily used among gentlemen. . . .\nOur Courtier then will be esteemed excellent and will attain grace in everything, particularly in speaking. . . .\nI would have him more than passably accomplished in letters, at least in those studies that are called the humanities, and conversant not only with the Latin language but with the Greek, for the sake of the many different things that have been admirably written therein. Let him be well versed in the poets, and not less in the orators and historians, and also proficient in writing verse and prose, especially in this vulgar [vernacular] tongue of ours. . . .\nYou must know that I am not content with the Courtier unless he be also a musician and unless, besides understanding and being able to read notes, he can play upon divers instruments. . . .\nI wish our Courtier to guard against getting the name of a liar or a boaster, which sometimes befalls even those who do not deserve it.\n\u2014From Baldassare Castiglione's Book of the Courtier, 1528\nWhich of the following reforms would Castiglione most likely want to institute?",
"choices": [
"The creation of coeducational public high schools paid for by the state",
"The creation of male-only public high schools paid for by the state",
"The creation of public universities without tuition where all students of both genders could learn",
"The creation of private high schools for elite young men with many extracurricular activities"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.27891206741333,
"B": -1.70829439163208,
"C": -1.2191929817199707,
"D": -1.4135003089904785
}
},
{
"ds_idx": 1077,
"subject": "college_computer_science",
"question": "Which of the following characteristics of a programming language is best specified using a context-free grammar?",
"choices": [
"Identifier length",
"Maximum level of nesting",
"Operator precedence",
"Type compatibility"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.3555864095687866,
"B": -1.3829129934310913,
"C": -1.1657003164291382,
"D": -1.7408822774887085
}
},
{
"ds_idx": 3203,
"subject": "high_school_computer_science",
"question": "What is the output of the statement \"a\" + \"ab\" in Python 3?",
"choices": [
"Error",
"aab",
"ab",
"a ab"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.4626214504241943,
"B": -1.3416192531585693,
"C": -1.4102590084075928,
"D": -1.362851858139038
}
},
{
"ds_idx": 3224,
"subject": "high_school_computer_science",
"question": "Consider the code segment below.\n Line 1: IF (a = 0)\n Line 2: {\n Line 3: b \u2190 a + 10\n Line 4: }\n Line 5: ELSE\n Line 6: {\n Line 7: b \u2190 a + 2O\n Line 8: }\n Which of the following changes will NOT affect the results when the code segment is executed?",
"choices": [
"Changing line 3 to b \u2190 10",
"Changing line 3 to a \u2190 b + 10",
"Changing line 7 to b \u2190 20",
"Changing line 7 to a \u2190 b + 10"
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.56117844581604,
"B": -1.5396177768707275,
"C": -1.4317991733551025,
"D": -1.1036245822906494
}
},
{
"ds_idx": 2978,
"subject": "high_school_biology",
"question": "A piece of potato is dropped into a beaker of pure water. Which of the following describes the activity after the potato is immersed into the water?",
"choices": [
"Water moves from the potato into the surrounding water.",
"Water moves from the surrounding water into the potato.",
"Potato cells plasmolyze.",
"Solutes in the water move into the potato."
],
"correct_idx": 1,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.3070578575134277,
"B": -1.3819117546081543,
"C": -1.2926039695739746,
"D": -1.6034255027770996
}
},
{
"ds_idx": 7336,
"subject": "medical_genetics",
"question": "Positional cloning refers to",
"choices": [
"using a selection procedure to clone a cDNA",
"cloning a portion of a gene using PCR",
"isolating a gene by PCR using primers from another species",
"mapping a gene to a chromosomal region and then identifying and cloning a genomic copy of the gene from the region"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.9546096324920654,
"B": -2.2911174297332764,
"C": -2.4754035472869873,
"D": -0.39911729097366333
}
},
{
"ds_idx": 241,
"subject": "astronomy",
"question": "Some of Mars'water is frozen in the soil and the ice caps and some was lost due to solar wind stripping but much of the original water is thought to have been lost through another mechanism. What happened to this \"lost\"water?",
"choices": [
"it was stripped away by the magnetic field when it rapidly decreased in strength",
"it was broken into hydrogen and oxygen by ultraviolet light and the hydrogen was lost through thermal escape",
"it was consumed by a civilization of thirsty Martians",
"it was vaporized during a period of intense volcanism and was lost to space through thermal escape"
],
"correct_idx": 1,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4465434551239014,
"B": -1.7164905071258545,
"C": -1.6072337627410889,
"D": -0.9732387661933899
}
},
{
"ds_idx": 802,
"subject": "college_biology",
"question": "Chemosynthetic bacteria can produce their own food despite their lack of photonabsorbing pigments because the chemosynthetic bacteria derive energy by",
"choices": [
"oxidizing inorganic substances such as ammonia, sulfur, or hydrogen",
"reducing inorganic substances such as ammonia, sulfur, or hydrogen",
"oxidizing organic compounds",
"transducing light energy into chemical energy"
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4409079551696777,
"B": -1.5062384605407715,
"C": -2.111274242401123,
"D": -0.8721938133239746
}
},
{
"ds_idx": 3430,
"subject": "high_school_european_history",
"question": "This question refers to the following information.\n\"XI. As the present sciences are useless for the discovery of effects, so the present system of logic is useless for the discovery of the sciences.\nXIX. There are and can exist but two ways of investigating and discovering truth. The one hurries on rapidly from the senses and particulars to the most general axioms, and from them, as principles and their supposed indisputable truth, derives and discovers the intermediate axioms. This is the way now in use. The other constructs its axioms from the senses and particulars, by ascending continually and gradually, till it finally arrives at the most general axioms, which is the true but unattempted way.\nXXII. Each of these two ways begins from the senses and particulars, and ends in the greatest generalities&\u2026\nXXXVI. We have but one simple method of delivering our sentiments, namely, we must bring men to particulars and their regular series and order, and they must for a while renounce their notions, and begin to form an acquaintance with things.\"\nFrancis Bacon, English philosopher and essayist, Novum Organum, 1620\nThe method of inquiry elucidated by Francis Bacon in the passage is known as",
"choices": [
"humanism",
"deduction",
"empiricism",
"scientific socialism"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.546999216079712,
"B": -1.1530344486236572,
"C": -1.12874436378479,
"D": -1.934032678604126
}
},
{
"ds_idx": 6391,
"subject": "human_sexuality",
"question": "Which of the following is typically associated with having a vasectomy?",
"choices": [
"decreased testosterone production",
"reduced rigidity of erections",
"decreased sperm production",
"normal orgasm"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4184921979904175,
"B": -1.4915589094161987,
"C": -0.9964264631271362,
"D": -1.8231419324874878
}
},
{
"ds_idx": 53,
"subject": "abstract_algebra",
"question": "Statement 1 | If G, H and K are groups of order 4, at least two of them are isomorphic. Statement 2 | If Z_r + Z_s is isomorphic to a subgroup Z_m + Z_n, then r divides m and s divides n.",
"choices": [
"True, True",
"False, False",
"True, False",
"False, True"
],
"correct_idx": 2,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.206595778465271,
"B": -1.3763707876205444,
"C": -1.5252164602279663,
"D": -1.5116418600082397
}
},
{
"ds_idx": 4266,
"subject": "high_school_mathematics",
"question": "What is the smallest positive integer with factors of 16, 15, and 12?",
"choices": [
"240",
"120",
"60",
"111"
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -1.25855553150177,
"B": -1.400544285774231,
"C": -1.4737502336502075,
"D": -1.4667712450027466
}
},
{
"ds_idx": 4730,
"subject": "high_school_microeconomics",
"question": "Which of the following is a characteristic of monopolistic competition in the long run?",
"choices": [
"Strong barriers to entry.",
"Homogeneous products.",
"Zero economic profits.",
"Minimum average total cost equals price."
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.466907024383545,
"B": -1.371814250946045,
"C": -1.47206449508667,
"D": -1.2603678703308105
}
},
{
"ds_idx": 6855,
"subject": "machine_learning",
"question": "The K-means algorithm:",
"choices": [
"Requires the dimension of the feature space to be no bigger than the number of samples",
"Has the smallest value of the objective function when K = 1",
"Minimizes the within class variance for a given number of clusters",
"Converges to the global optimum if and only if the initial means are chosen as some of the samples themselves"
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5917282104492188,
"B": -1.596639633178711,
"C": -1.5074329376220703,
"D": -1.00067138671875
}
},
{
"ds_idx": 6452,
"subject": "international_law",
"question": "Which of the following is a treaty-based human rights mechanism?",
"choices": [
"The UN Human Rights Committee",
"The UN Human Rights Council",
"The UN Universal Periodic Review",
"The UN special mandates"
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -1.1254675388336182,
"B": -1.2834742069244385,
"C": -1.2416212558746338,
"D": -2.2435762882232666
}
},
{
"ds_idx": 3532,
"subject": "high_school_geography",
"question": "What family do most European languages belong to?",
"choices": [
"Ural-Altaic",
"Basque",
"Indo-European",
"Phoenician"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.6779412031173706,
"B": -2.2564399242401123,
"C": -0.4544762074947357,
"D": -2.696017026901245
}
},
{
"ds_idx": 3639,
"subject": "high_school_geography",
"question": "What are the two major branches of Islam?",
"choices": [
"Sunni and Shinto",
"Shinto and Shiite",
"Sunni and Shiite",
"Shamanism and Shiite"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.4718633890151978,
"B": -1.6716924905776978,
"C": -0.7680479288101196,
"D": -2.1934304237365723
}
},
{
"ds_idx": 2577,
"subject": "global_facts",
"question": "As of 2017, what fraction of the population in Brazil used the internet in the past three months?",
"choices": [
"18%",
"38%",
"58%",
"78%"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.6890255212783813,
"B": -1.2549797296524048,
"C": -1.1767841577529907,
"D": -1.53630793094635
}
},
{
"ds_idx": 2477,
"subject": "formal_logic",
"question": "Select the best translation into predicate logic. Holly loves Frances. (f: Frances; h: Holly; Lxy: x loves y)",
"choices": [
"hLf",
"fLh",
"Lfh",
"Lhf"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.4391639232635498,
"B": -1.5805175304412842,
"C": -1.5467231273651123,
"D": -1.0874755382537842
}
},
{
"ds_idx": 13288,
"subject": "security_studies",
"question": "What role does 'Information' play in the economic science of decision making?",
"choices": [
"Information is not important to decision making in economic theory; rather, it is an actor's perceptions of situations that will inform their decision making process.",
"Information has no special significance to economic decision making, which is guided more by external forces that coerce actors than their own rational choices.",
"Information plays a significant role in decision making because, although theory asserts that actors act in their own best interest, 'rational action' depends on the information available to them at the time.",
"Information has become an important resource in the 21st century and can be sold and traded as a market commodity."
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.31885826587677,
"B": -1.6302672624588013,
"C": -1.039002537727356,
"D": -1.7195903062820435
}
},
{
"ds_idx": 1584,
"subject": "conceptual_physics",
"question": "As a blinking light source approaching you gains speed, you see the frequency of flashes",
"choices": [
"increase.",
"decrease.",
"remain unchanged.",
"None of these."
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.1894301176071167,
"B": -1.2657774686813354,
"C": -1.0694369077682495,
"D": -2.7081661224365234
}
},
{
"ds_idx": 468,
"subject": "business_ethics",
"question": "The focus on issues, the intended audience extending beyond shareholders and the voluntary nature of ___________ distinguish it from _________.",
"choices": [
"Financial accounting, Environmental management initiatives",
"Financial accounting, Social accounting.",
"Social accounting, Financial accounting.",
"Annual report publication, Social accounting"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.578283667564392,
"B": -1.219997763633728,
"C": -1.2538608312606812,
"D": -1.5561126470565796
}
},
{
"ds_idx": 6504,
"subject": "international_law",
"question": "What is passive personality jurisdiction?",
"choices": [
"It is jurisdiction based on the nationality of the offender",
"It is jurisdiction based on where the offence was committed",
"It is jurisdiction based on the nationality of the victims",
"It is jurisdiction based on the country where the legal person was Registered"
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.3539457321166992,
"B": -0.9404974579811096,
"C": -1.6814165115356445,
"D": -1.8315439224243164
}
},
{
"ds_idx": 10437,
"subject": "professional_accounting",
"question": "A spouse died on December 31, year 1. The couple had no dependents. What should be the filing status of the surviving spouse in year 2?",
"choices": [
"Single.",
"Married filing jointly.",
"Qualifying widow(er).",
"Head of household."
],
"correct_idx": 0,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.5424282550811768,
"B": -1.2455799579620361,
"C": -1.1770298480987549,
"D": -1.7100918292999268
}
},
{
"ds_idx": 427,
"subject": "business_ethics",
"question": "Demand reduction.",
"choices": [
"1,3,4",
"2,3,4",
"1,2,3",
"1,2,4"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.5427181720733643,
"B": -1.5857765674591064,
"C": -1.3304016590118408,
"D": -1.1606457233428955
}
},
{
"ds_idx": 4880,
"subject": "high_school_physics",
"question": "An electric dipole consists of a pair of equal but opposite point charges of magnitude 4.0 nC separated by a distance of 2.0 cm. What is the electric field strength at the point midway between the charges?",
"choices": [
"0",
"9.0 \u00d7 10^4 V/m",
"1.8 \u00d7 10^5 V/m",
"7.2 \u00d7 10^5 V/m"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.7228232622146606,
"B": -1.090140461921692,
"C": -1.3589354753494263,
"D": -1.517670750617981
}
},
{
"ds_idx": 1004,
"subject": "college_computer_science",
"question": "Which of the following statements about floating-point arithmetic is NOT true?",
"choices": [
"It is inherently nonassociative because some numbers have no exact representation.",
"It is inherently nonassociative because there have to be upper and lower bounds on the size of numbers.",
"Associativity can be achieved with appropriate roundoff conventions.",
"Some rational numbers have no exact representation."
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5607290267944336,
"B": -1.4202775955200195,
"C": -1.5540342330932617,
"D": -1.1055173873901367
}
},
{
"ds_idx": 13003,
"subject": "professional_psychology",
"question": "Object relations therapy is similar to Freudian psychodynamic therapy in its primary focus on",
"choices": [
"issues of attachment and separation",
"the neutral role of the therapist",
"early childhood experiences",
"the ego, id, and superego"
],
"correct_idx": 2,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.2103525400161743,
"B": -1.582628846168518,
"C": -1.3133283853530884,
"D": -1.4880396127700806
}
},
{
"ds_idx": 4360,
"subject": "high_school_mathematics",
"question": "For all real numbers $r$ and $s$, define the mathematical operation $\\#$ such that the following conditions apply: $r\\ \\#\\ 0 = r, r\\ \\#\\ s = s\\ \\#\\ r$, and $(r + 1)\\ \\#\\ s = (r\\ \\#\\ s) + s + 1$. What is the value of $11\\ \\#\\ 5$?",
"choices": [
"11",
"12",
"64",
"71"
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.3579834699630737,
"B": -1.3218430280685425,
"C": -1.4177502393722534,
"D": -1.564228892326355
}
},
{
"ds_idx": 13789,
"subject": "virology",
"question": "Many virus infections result in immune pathology, this is caused predominantly by which of the following?",
"choices": [
"A cytokine storm",
"Antibody mediated reactions",
"A response of killer T cells (CD-8)",
"A decline in function of the immune system"
],
"correct_idx": 0,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.225774884223938,
"B": -1.041174054145813,
"C": -1.7736760377883911,
"D": -1.715652585029602
}
},
{
"ds_idx": 8153,
"subject": "miscellaneous",
"question": "Ada Lovelace is credited with being the first person to have made what?",
"choices": [
"a computer program",
"a souffle",
"a brassiere",
"a mystery novel"
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -0.4479084014892578,
"B": -2.1522369384765625,
"C": -2.4869213104248047,
"D": -1.8460731506347656
}
},
{
"ds_idx": 1921,
"subject": "electrical_engineering",
"question": "Potentiometer method of DC voltage measurement is more accurate than direct measurement using a voltmeter because",
"choices": [
"It loads the circuit moderately.",
"It loads the circuit to maximum extent.",
"It uses centre zero galvanometer instead of voltmeter.",
"It does not load the circuit at all."
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.398123025894165,
"B": -1.5016100406646729,
"C": -1.2173960208892822,
"D": -1.463075876235962
}
},
{
"ds_idx": 11550,
"subject": "professional_law",
"question": "A witness is scheduled to testify in a criminal case. To prove that he was a juvenile on a given date, evidence is offered that on that date he was confined in a juvenile detention facility. If a party wished to argue that this evidence is hearsay, whom would he point to as the hearsay declarant?",
"choices": [
"The witness on the stand.",
"The party offering the evidence.",
"The juvenile authorities.",
"The witness."
],
"correct_idx": 2,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.4554983377456665,
"B": -1.055763840675354,
"C": -1.848833680152893,
"D": -1.3498483896255493
}
},
{
"ds_idx": 6494,
"subject": "international_law",
"question": "How are the members of the arbitral tribunal appointed?",
"choices": [
"All the members of the arbitral tribunal are appointed by the parties",
"All the members of the arbitral tribunal are appointed by the parties",
"All the members of the arbitral tribunal are appointed by an impartial third party, such as the president of the ICJ",
"All the members of the arbitral tribunal are appointed by the parties from a restricted list of arbitrators"
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.5178974866867065,
"B": -1.6287506818771362,
"C": -1.5524567365646362,
"D": -1.0006836652755737
}
},
{
"ds_idx": 5482,
"subject": "high_school_statistics",
"question": "A college admissions officer is interested in comparing the SAT math scores of high school applicants who have and have not taken AP Statistics. She randomly pulls the files of five applicants who took AP Statistics and five applicants who did not, and proceeds to run a t-test to compare the mean SAT math scores of the two groups. Which of the following is a necessary assumption?",
"choices": [
"The population variances from each group are known.",
"The population variances from each group are unknown.",
"The population variances from the two groups are equal.",
"The population of SAT scores from each group is normally distributed."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.5029566287994385,
"B": -1.246279001235962,
"C": -1.7152483463287354,
"D": -1.1761305332183838
}
},
{
"ds_idx": 171,
"subject": "anatomy",
"question": "Which of the following is a substance that aids the transmission of nerve impulses to the muscles?",
"choices": [
"Acetylcholine",
"Cholecystokinin",
"Deoxyribose",
"Oxytocin"
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -0.8313019275665283,
"B": -1.4833920001983643,
"C": -2.0255672931671143,
"D": -1.5985996723175049
}
},
{
"ds_idx": 13378,
"subject": "security_studies",
"question": "Which of the following does not constitute a fundamental ontological principle of social constructivism?",
"choices": [
"Normative or ideational structures are important and matter as much as, if not more than, material structures.",
"Identities are important.",
"Anarchy is an inescapable feature of the international system.",
"Agents and structures are mutually constituted."
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.435605764389038,
"B": -1.7287099361419678,
"C": -1.4396111965179443,
"D": -1.0664193630218506
}
},
{
"ds_idx": 13606,
"subject": "us_foreign_policy",
"question": "What was the significance of the Gulf of Tonkin resolution?",
"choices": [
"It allowed the US to intensify its involvement in Vietnam",
"It illustrated the influence of public opinion on US foreign policy",
"It enhanced Congressional control over the Vietnam War",
"It curtailed US involvement in Vietnam"
],
"correct_idx": 0,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.462350606918335,
"B": -1.914203405380249,
"C": -1.7816369533538818,
"D": -0.7997339367866516
}
},
{
"ds_idx": 922,
"subject": "college_chemistry",
"question": "Which of the following must be true about a binary liquid mixture that obeys Raoult\u2019s law?\nI. The partial pressure of each component at equilibrium is proportional to its mole fraction in the liquid mixture.\nII. The volume of the mixture is equal to the sum of the volumes of each component before mixing.\nIII. Intermolecular interactions in the mixture are identical to intermolecular interactions in the pure components.",
"choices": [
"I only",
"III only",
"I and III only",
"I, II, and III"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.9294759035110474,
"B": -2.221118927001953,
"C": -1.0173379182815552,
"D": -0.9743729829788208
}
},
{
"ds_idx": 12432,
"subject": "professional_medicine",
"question": "A 64-year-old female presents to the office with the complaint of difficulty swallowing saliva for about 6\u00a0months. She has slurred speech. Examination of the tongue reveals wasting and fasciculation. Decreased deep tendon reflexes and fasciculation are noted in the left lower extremity. The deep tendon reflexes of the left upper extremity are hyperactive. The most likely diagnosis is",
"choices": [
"amyotrophic lateral sclerosis",
"Friedreich ataxia",
"Pick disease",
"progressive multifocal leukoencephalopathy"
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -0.8140345811843872,
"B": -1.59174644947052,
"C": -1.8625651597976685,
"D": -1.6295768022537231
}
},
{
"ds_idx": 1851,
"subject": "econometrics",
"question": "Suppose that the following regression is estimated using 27 quarterly observations:\n\n$y_t = \\beta_1 + \\beta_2 x_2 + \\beta_3 x_{3t} + u_t$\n\nWhat is the appropriate critical value for a 2-sided 5% size of test of $H_0: \\beta_3 = 1$?",
"choices": [
"1.64",
"1.71",
"2.06",
"1.96"
],
"correct_idx": 2,
"picked_idx": 3,
"is_correct": false,
"letter_log_probs": {
"A": -1.4860291481018066,
"B": -1.4334321022033691,
"C": -1.3964905738830566,
"D": -1.2567734718322754
}
},
{
"ds_idx": 7261,
"subject": "marketing",
"question": " Which technique is used to collect data that has been previously collected for a purpose other than the current research situation?",
"choices": [
"Primary research.",
"Secondary research.",
"Primary and secondary.",
"Primary and desk research."
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.382755994796753,
"B": -1.2330119609832764,
"C": -1.3760039806365967,
"D": -1.5938231945037842
}
},
{
"ds_idx": 3043,
"subject": "high_school_chemistry",
"question": "Which of the following is the strongest Br\u00f8nsted-Lowry acid?",
"choices": [
"HBrO",
"HBrO3",
"HBrO2",
"HBrO4"
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.4570022821426392,
"B": -1.4703251123428345,
"C": -1.432458519935608,
"D": -1.224378228187561
}
},
{
"ds_idx": 9882,
"subject": "philosophy",
"question": "According to Gauthier, deliberative justification:",
"choices": [
"is refuted by moral requirements.",
"is replaced once we acknowledge moral requirements.",
"refutes morality.",
"ignores morality and seemingly replaces it."
],
"correct_idx": 3,
"picked_idx": 1,
"is_correct": false,
"letter_log_probs": {
"A": -1.340877652168274,
"B": -1.165899395942688,
"C": -1.4607011079788208,
"D": -1.665563702583313
}
},
{
"ds_idx": 5676,
"subject": "high_school_us_history",
"question": "This question refers to the following information.\n\"An act for the more effectual protection of the property of married women:\n\"\u00a71. The real property of any female who may hereafter marry, and which she shall own at the time of marriage, and the rents, issues, and profits thereof, shall not be subject to the sole disposal of her husband, nor be liable for his debts, and shall continue her sole and separate property, as if she were a single female.\n\"\u00a72. The real and personal property, and the rents, issues, and profits thereof, of any female now married, shall not be subject to the disposal of her husband; but shall be her sole and separate property, as if she were a single female, except so far as the same may be liable for the debts of her husband heretofore contracted.\n\"\u00a73. Any married female may take by inheritance, or by gift, grant, devise, or bequest, from any person other than her husband, and hold to her sole and separate use, and convey and devise real and personal property, and any interest or estate therein, and the rents, issues, and profits thereof, in the same manner and with like effect as if she were unmarried, and the same shall not be subject to the disposal of her husband nor be liable for his debts.\"\n\u2014Married Women's Property Act, New York State (1848)\nThe ideas expressed in the passage above most directly reflect which of the following continuities in U.S. history?",
"choices": [
"Debates about access to voting rights.",
"Debates about the role of federal government in marriage law.",
"Debates about discrimination in employment.",
"Debates about the legal status of women."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.803696870803833,
"B": -1.4239704608917236,
"C": -1.7892391681671143,
"D": -0.8563149571418762
}
},
{
"ds_idx": 1011,
"subject": "college_computer_science",
"question": "For all strings x, the function x^M is defined recursively as follows.\ne^M= e, and\nif w is a string and a is a string with length 1, then\n(aw)^M = aw^Ma.\nLet a be a string with length 1, and let x and y be arbitrary strings. Which of the following is true?",
"choices": [
"a^M = a",
"(ax)^M = (xa)^M",
"(xy)^M = y^Mx^M",
"None of the above"
],
"correct_idx": 3,
"picked_idx": 0,
"is_correct": false,
"letter_log_probs": {
"A": -1.338955044746399,
"B": -1.4213944673538208,
"C": -1.414098858833313,
"D": -1.4842740297317505
}
},
{
"ds_idx": 128,
"subject": "anatomy",
"question": "A patient sustains severe blunt trauma to the left upper abdomen and requires surgery. Which one of the following organs is most likely to be involved?",
"choices": [
"Appendix",
"Gallbladder",
"Spleen",
"Urinary bladder"
],
"correct_idx": 2,
"picked_idx": 2,
"is_correct": true,
"letter_log_probs": {
"A": -1.7165191173553467,
"B": -1.7030818462371826,
"C": -0.7089202404022217,
"D": -1.9627788066864014
}
},
{
"ds_idx": 46,
"subject": "abstract_algebra",
"question": "Statement 1 | A homomorphism may have an empty kernel. Statement 2 | It is not possible to have a nontrivial homomorphism of some finite group into some infinite group.",
"choices": [
"True, True",
"False, False",
"True, False",
"False, True"
],
"correct_idx": 1,
"picked_idx": 1,
"is_correct": true,
"letter_log_probs": {
"A": -1.352242350578308,
"B": -1.2415951490402222,
"C": -1.4942177534103394,
"D": -1.52415931224823
}
},
{
"ds_idx": 9624,
"subject": "nutrition",
"question": "Molecules of natural polyunsaturated fatty acids in vegetable oils contain:\n",
"choices": [
"18 carbon atoms with one carbon-carbon double bond in the cis configuration",
"20 carbon atoms with at least two carbon-carbon double bonds in the cis configuration",
"18 carbon atoms with at least two carbon-carbon double bonds in the trans configuration",
"18 carbon atoms with at least two carbon-carbon double bonds in the cis configuration"
],
"correct_idx": 3,
"picked_idx": 2,
"is_correct": false,
"letter_log_probs": {
"A": -1.4867055416107178,
"B": -1.3564393520355225,
"C": -1.1435163021087646,
"D": -1.6399285793304443
}
},
{
"ds_idx": 8329,
"subject": "moral_disputes",
"question": " To say that an action is intrinsically permissible (a feature mentioned in the doctrine of double effect) is to say that",
"choices": [
"the action, apart from its effects, is morally permissible.",
"the action, only because of its effects, is morally permissible.",
"no actual person would ever endorse that action.",
"the action is valuable because of its relationship to things that are intrinsically valuable."
],
"correct_idx": 0,
"picked_idx": 0,
"is_correct": true,
"letter_log_probs": {
"A": -1.1705375909805298,
"B": -1.1972309350967407,
"C": -1.6244865655899048,
"D": -1.6965347528457642
}
},
{
"ds_idx": 6419,
"subject": "human_sexuality",
"question": "Which is true with regard to voyeurs?",
"choices": [
"They are generally middle-aged.",
"They are usually dangerous.",
"They typically want to view friends' sexual behaviours.",
"They typically want to observe strangers' activities."
],
"correct_idx": 3,
"picked_idx": 3,
"is_correct": true,
"letter_log_probs": {
"A": -1.6031876802444458,
"B": -1.5718919038772583,
"C": -1.3879472017288208,
"D": -1.0807801485061646
}
}
],
"by_subject": {
"formal_logic": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"high_school_macroeconomics": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"management": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"high_school_physics": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"high_school_statistics": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"security_studies": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"jurisprudence": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"sociology": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"high_school_government_and_politics": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"elementary_mathematics": {
"correct": 0,
"total": 4,
"accuracy": 0.0
},
"abstract_algebra": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"high_school_mathematics": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"clinical_knowledge": {
"correct": 0,
"total": 4,
"accuracy": 0.0
},
"electrical_engineering": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"world_religions": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"medical_genetics": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"conceptual_physics": {
"correct": 0,
"total": 4,
"accuracy": 0.0
},
"high_school_us_history": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"global_facts": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"professional_psychology": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"computer_security": {
"correct": 0,
"total": 4,
"accuracy": 0.0
},
"us_foreign_policy": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"college_mathematics": {
"correct": 0,
"total": 4,
"accuracy": 0.0
},
"high_school_chemistry": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"public_relations": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"high_school_microeconomics": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"miscellaneous": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"high_school_european_history": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"anatomy": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"virology": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"prehistory": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"philosophy": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"college_biology": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"econometrics": {
"correct": 0,
"total": 4,
"accuracy": 0.0
},
"college_computer_science": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"high_school_world_history": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"high_school_biology": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"high_school_geography": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"professional_law": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"logical_fallacies": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"business_ethics": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"moral_scenarios": {
"correct": 0,
"total": 4,
"accuracy": 0.0
},
"marketing": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"college_physics": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"college_chemistry": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"astronomy": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"machine_learning": {
"correct": 0,
"total": 4,
"accuracy": 0.0
},
"high_school_psychology": {
"correct": 3,
"total": 4,
"accuracy": 0.75
},
"nutrition": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"human_aging": {
"correct": 1,
"total": 4,
"accuracy": 0.25
},
"high_school_computer_science": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"professional_accounting": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"college_medicine": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"professional_medicine": {
"correct": 4,
"total": 4,
"accuracy": 1.0
},
"moral_disputes": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"international_law": {
"correct": 2,
"total": 4,
"accuracy": 0.5
},
"human_sexuality": {
"correct": 1,
"total": 4,
"accuracy": 0.25
}
},
"wall_time_seconds": 903.4679901599884
}
Reference in New Issue View Git Blame Copy Permalink