From e313f32a5d96a31eac66772630d6590a1197cb6d Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 26 Sep 2026 21:34:42 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: appvoid/palmer-006 Source: Original Platform --- .gitattributes | 45 + FALCON_LICENSE | 265 + LICENSE | 199 + NOTICE | 22 + README.md | 494 + completion.py | 120 + config.json | 70 + generation_config.json | 7 + images/006.png | 3 + images/1.png | Bin 0 -> 8565 bytes images/2.png | Bin 0 -> 12515 bytes images/3.png | Bin 0 -> 16936 bytes images/4.png | Bin 0 -> 9014 bytes images/benchmarks.png | 3 + model.gguf | 3 + model.safetensors | 3 + prompt.txt | 5 + ...ppvoid_palmer-006_arithmark-3_results.json | 157 + reports/bananamid_base_1_1_report.json | 19763 ++ reports/palmer-006_lm_eval_harness.log | 94 + special_tokens_map.json | 342 + tokenizer.json | 165548 +++++++++++++++ tokenizer_config.json | 4534 + 23 files changed, 191677 insertions(+) create mode 100644 .gitattributes create mode 100644 FALCON_LICENSE create mode 100644 LICENSE create mode 100644 NOTICE create mode 100644 README.md create mode 100644 completion.py create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 images/006.png create mode 100644 images/1.png create mode 100644 images/2.png create mode 100644 images/3.png create mode 100644 images/4.png create mode 100644 images/benchmarks.png create mode 100644 model.gguf create mode 100644 model.safetensors create mode 100644 prompt.txt create mode 100644 reports/appvoid_palmer-006_arithmark-3_results.json create mode 100644 reports/bananamid_base_1_1_report.json create mode 100644 reports/palmer-006_lm_eval_harness.log create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ae2a95f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,45 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +images/001.png filter=lfs diff=lfs merge=lfs -text +images/002.png filter=lfs diff=lfs merge=lfs -text +images/003.png filter=lfs diff=lfs merge=lfs -text +images/004.png filter=lfs diff=lfs merge=lfs -text +images/005.png filter=lfs diff=lfs merge=lfs -text +images/006.png filter=lfs diff=lfs merge=lfs -text +images/007.png filter=lfs diff=lfs merge=lfs -text +images/008.png filter=lfs diff=lfs merge=lfs -text +benchmarks.png filter=lfs diff=lfs merge=lfs -text +model.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/FALCON_LICENSE b/FALCON_LICENSE new file mode 100644 index 0000000..48159cd --- /dev/null +++ b/FALCON_LICENSE @@ -0,0 +1,265 @@ +TII FALCON LICENSE +December 2024 + +Official source: +https://falconllm.tii.ae/falcon-terms-and-conditions.html + +Introductory note + +This license is, in part, based on the Apache License Version 2.0 +(available at http://www.apache.org/licenses/), with a series of modifications. +The contribution of the Apache License 2.0 to the framing of this document is +acknowledged. + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions + +“License” shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 to 11 of this document. + +“Licensor” shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +“Legal Entity” shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, “control” means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +“You” (or “Your”) shall mean an individual or Legal Entity exercising +permissions granted by this License. + +“Source” form shall mean the preferred form for making modifications, including +but not limited to software source code, training datasets used for training or +fine tuning a machine learning model or artificial intelligence model, +documentation source, and configuration files. + +“Object” form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +a trained and/or fine-tuned machine learning model or artificial intelligence +model, generated documentation, and conversions to other media types. + +“Work” shall mean the work of authorship, which in relation to the initial +release of each Relevant Model is in Object form only, but in the case of any +and all Derivative Works means that Work whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +“Derivative Works” shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +“Contribution” shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. + +For the purposes of this definition, “submitted” means any form of electronic, +verbal, or written communication sent to the Licensor or its representatives, +including but not limited to communication on electronic mailing lists, source +code control systems, and issue tracking systems that are managed by, or on +behalf of, the Licensor for the purpose of discussing and improving the Work, +but excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as “Not a Contribution.” + +“Contributor” shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +“Acceptable Use Policy” means the latest version from time to time of the policy +designated as such hosted at FalconLLM.tii.ae. + +“Relevant Model” shall mean those artificial intelligence models to which this +license has been applied (as indicated by a copyright notice in the form stated +in the Appendix below), each initially made available in Object form only. + +“TII” shall mean the Technology Innovation Institute – Sole Proprietorship +L.L.C., or any party nominated in writing by Technology Innovation Institute – +Sole Proprietorship L.L.C. as its successor for the purposes of this License, or +any party nominated in writing to be a successor to any successor for the +purposes of this license. + +2. Grant of Copyright License + +2.1 Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, irrevocable copyright +license to reproduce, prepare Derivative Works of, publicly display, publicly +perform, sublicense, and distribute the Work and such Derivative Works in Source +or Object form. + +2.2 Your copyright license to use the Work shall be royalty free and without +charge. + +3. Grant of Patent License + +3.1 Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, irrevocable (except as +stated in this section) patent license to make, have made, use, offer to sell, +sell, import, and otherwise transfer the Work, where such license applies only +to those patent claims licensable by such Contributor that are necessarily +infringed by their Contribution(s) alone or by combination of their +Contribution(s) with the Work to which such Contribution(s) was submitted. + +If You institute patent litigation against any entity (including a cross-claim +or counterclaim in a lawsuit) alleging that the Work or a Contribution +incorporated within the Work constitutes direct or contributory patent +infringement, then any patent licenses granted to You under this License for +that Work shall terminate as of the date such litigation is filed. + +3.2 Your patent license to use the Work shall be royalty free and without +charge. + +4. Redistribution + +4.1 You may reproduce and distribute copies of the Work or Derivative Works +thereof in any medium, with or without modifications, and in Source or Object +form, provided that You meet the following conditions: + +4.1.1 Use-based restrictions incorporating the Acceptable Use Policy in the +manner set out in Section 5 of this license, and which do not otherwise conflict +with the Acceptable Use Policy, must be included as enforceable provisions by +You in any type of legal agreement (e.g. a license) governing the use and/or +distribution of the Work or any Derivative Works that You distribute; + +4.1.2 You must give any other recipients of the Work or Derivative Works a copy +of this License; + +4.1.3 You must cause any modified files to carry prominent notices stating that +You changed the files; + +4.1.4 You must retain, in the Source form of any Derivative Works that You +distribute, all copyright, patent, trademark, and attribution notices from the +Source form of the Work, excluding those notices that do not pertain to any part +of the Derivative Works; and + +4.1.5 If the Work includes a “NOTICE” text file as part of its distribution, +then any Derivative Works that You distribute must include a readable copy of +the attribution notices contained within such NOTICE file, excluding those +notices that do not pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. + +The contents of the NOTICE file are for informational purposes only and do not +modify the License. You may add Your own attribution notices within Derivative +Works that You distribute, alongside or as an addendum to the NOTICE text from +the Work, provided that such additional attribution notices cannot be construed +as modifying the License. + +4.2 Subject to Section 6, You may add Your own copyright statement to Your +modifications and may provide additional or different license terms and +conditions for use, reproduction, or distribution of Your modifications, or for +any such Derivative Works as a whole, provided Your use, reproduction, and +distribution of the Work otherwise complies with the conditions stated in this +License. + +5. Acceptable Use + +5.1 Subject to Section 5.3, your use of the Work or any Derivative Work must +comply with the Acceptable Use Policy at all times. You shall procure that all +persons using the Work or Derivative Work for you or on your behalf comply with +the Acceptable Use Policy in their use. + +5.2 You may not use the Work or any Derivative Work or any output from the Work +or Derivative Work, whether directly or indirectly, to create other works for +any purpose which conflicts with the Acceptable Use Policy. + +5.3 The Acceptable Use Policy may be updated from time to time. You should +monitor the web address at which the Acceptable Use Policy is hosted to ensure +that your use of the Work or any Derivative Work complies with the updated +Acceptable Use Policy. + +6. Publication + +6.1 You shall include prominently in any public statement regarding a +Derivative Work the following statement: + +“[name of relevant Derivative Work] is built using artificial intelligence +technology from the Technology Innovation Institute”. + +6.2 You may request from TII a reasonably adjusted version of the above +statement to suit the publication the relevant statement is being made in. TII +will not unreasonably withhold or delay approval of such a request. + +7. Submission of Contributions + +7.1 Unless You explicitly state otherwise, any Contribution intentionally +submitted for inclusion in the Work by You to the Licensor shall be under the +terms and conditions of this License, without any additional terms or +conditions. + +7.2 Notwithstanding the above, nothing herein shall supersede or modify the +terms of any separate license agreement you may have executed with Licensor +regarding such Contributions. + +8. Trademarks + +8.1 Except as required for compliance with Section 6 of this License, this +License does not grant permission to use the trade names, trademarks, service +marks, or product names of the Licensor, except as required for reasonable and +customary use in describing the origin of the Work and reproducing the content +of the NOTICE file. + +9. Disclaimer of Warranty + +9.1 Unless required by applicable law or agreed to in writing, Licensor provides +the Work (and each Contributor provides its Contributions) on an “AS IS” BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + +You are solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +10. Limitation of Liability + +10.1 In no event and under no legal theory, whether in tort (including +negligence), contract, or otherwise, unless required by applicable law (such as +deliberate and grossly negligent acts) or agreed to in writing, shall any +Contributor be liable to You for damages, including any direct, indirect, +special, incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the Work (including +but not limited to damages for loss of goodwill, work stoppage, computer failure +or malfunction, or any and all other commercial damages or losses), even if +such Contributor has been advised of the possibility of such damages. + +11. Accepting Warranty or Additional Liability + +11.1 While redistributing the Work or Derivative Works thereof, You may choose +to offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. + +However, in accepting such obligations, You may act only on Your own behalf and +on Your sole responsibility, not on behalf of any other Contributor, and only +if You agree to indemnify, defend, and hold each Contributor harmless for any +liability incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the TII Falcon License to your work + +To apply the TII Falcon License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets “[]” replaced with your own +identifying information. Do not include the brackets. The text should be +enclosed in the appropriate comment syntax for the file format. + +Copyright [yyyy] [name of copyright owner] +Licensed under the TII Falcon License (the “License”); you may not use this file +except in compliance with the License. + +You may obtain a copy of the License at FalconLLM.tii.ae. Unless required by +applicable law or agreed to in writing, software distributed under the License +is distributed on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. + +See the License for the specific language governing permissions and limitations +under the License. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c7e02ba --- /dev/null +++ b/LICENSE @@ -0,0 +1,199 @@ +APPVOID NONCOMMERCIAL MODEL LICENSE +Version 1.0 — 26 July 2026 + +Copyright (c) 2026 appvoid. All rights reserved. + +NOTICE: This is a source-available, noncommercial model license. It is not an open-source license as that term is commonly defined. Commercial rights are available only under a separate written agreement with appvoid. + +By downloading, accessing, copying, using, modifying, fine-tuning, converting, hosting, or distributing any Model Materials, You accept this License. + +1. DEFINITIONS + +1.1 “Licensor” means appvoid, to the extent appvoid owns or is authorized to license the applicable rights in the Model Materials. + +1.2 “Model Materials” means the machine-learning materials identified as being released under this License, including model weights, checkpoints, parameter files, adapters, LoRAs, quantizations, configuration files, tokenizer files, inference files, and accompanying documentation, but excluding Third-Party Materials. + +1.3 “Model” means the machine-learning model embodied in or created from the Model Materials. + +1.4 “Derivative Model” means any model, checkpoint, adapter, quantization, conversion, merge, fine-tune, distillation, modification, or other work based on, derived from, incorporating, or trained using the Model or Model Materials, where appvoid’s protectable contributions remain present. Ordinary Outputs are not Derivative Models solely because they were generated by the Model. + +1.5 “Output” means text, code, data, embeddings, or other content generated by running the Model, excluding any portion that reproduces Model Materials or other protected material. + +1.6 “Noncommercial Purpose” means use that is not primarily intended for, and does not result in, commercial advantage, monetary compensation, revenue, business operational benefit, or support for a commercial product or service. Permitted examples include: + +(a) personal study, experimentation, hobby projects, and private research; +(b) academic teaching or academic research; +(c) use by a charitable or nonprofit organization for its public-interest mission; +(d) use by a public educational or public research institution; and +(e) publication of noncommercial research results. + +A use is not Noncommercial merely because no fee is charged. + +1.7 “Commercial Use” means any use that is not a Noncommercial Purpose, including, without limitation: + +(a) use by or for a for-profit entity, except the limited evaluation permitted in Section 3; +(b) use in a paid, subscription-based, advertising-supported, sponsored, or revenue-generating product or service; +(c) use in internal business operations, employee tools, analytics, customer support, automation, research and development, or decision-making; +(d) providing hosted inference, fine-tuning, training, model access, an API, or managed services to another person or entity; +(e) creating, training, evaluating, or improving a model intended for commercial deployment; +(f) consulting, contracting, client work, or creation of commercial deliverables; +(g) sale, rental, paid licensing, paid redistribution, or other commercial exploitation of the Model Materials or any Derivative Model; or +(h) generating Outputs for a commercial product, service, client, campaign, or business activity. + +1.8 “Distribute” means to provide, publish, transmit, sublicense, transfer, or otherwise make available a copy of Model Materials or a Derivative Model to another person or entity. + +1.9 “Third-Party Materials” means any component governed by a license, copyright, patent, trademark, acceptable-use policy, or other right belonging to a party other than appvoid. + +1.10 “You” means the individual or legal entity exercising rights under this License. + +2. NONCOMMERCIAL LICENSE GRANT + +Subject to full compliance with this License and all applicable Third-Party Terms, appvoid grants You a worldwide, non-exclusive, royalty-free, non-transferable license, for Noncommercial Purposes only, to: + +(a) access, use, run, reproduce, and evaluate the Model Materials; +(b) modify, fine-tune, merge, distill, quantize, convert, and create Derivative Models; +(c) publicly display and publish noncommercial research concerning the Model; and +(d) Distribute the Model Materials or Derivative Models as permitted by Section 4. + +This grant is perpetual unless terminated under Section 10. No rights are granted for Commercial Use. + +To the extent appvoid owns patent claims necessarily infringed by exercising the permissions expressly granted above, appvoid grants You a corresponding non-exclusive, worldwide, royalty-free patent license for those permitted Noncommercial Purposes only. + +3. LIMITED COMMERCIAL EVALUATION + +A for-profit entity may evaluate the Model internally for up to thirty (30) calendar days solely to decide whether to obtain a commercial license from appvoid, provided that: + +(a) the Model is not used in production; +(b) no external user, customer, client, or partner is given access; +(c) no revenue, commercial deliverable, operational decision, or business service is produced from the evaluation; +(d) no personal, confidential, regulated, or customer data is used; +(e) no Derivative Model is deployed or retained for later commercial use; and +(f) all use stops at the end of the evaluation period unless appvoid grants a separate written license. + +This limited evaluation permission does not authorize benchmarking as a paid service, competitive model training, or use for the benefit of a third party. + +4. DISTRIBUTION AND DERIVATIVE MODELS + +You may Distribute Model Materials or Derivative Models only for Noncommercial Purposes and only if You: + +(a) provide every recipient with a complete copy of this License; +(b) license the distributed Model Materials and Derivative Models under this same License, without removing or weakening its noncommercial restriction; +(c) preserve all copyright, patent, trademark, attribution, and license notices; +(d) prominently identify files or materials You changed and describe the general nature of those changes; +(e) include all applicable Third-Party licenses, notices, and acceptable-use terms; +(f) do not state or imply that appvoid endorses You, Your work, or Your Derivative Model; and +(g) do not impose terms that conflict with this License or applicable Third-Party Terms. + +You may charge only the reasonable, direct cost of physical media or file transfer. You may not charge a license fee, access fee, hosting fee, support fee, or other compensation in connection with distribution under this License. + +The permissions in this Section do not authorize You to sublicense appvoid’s rights under different terms. + +5. COMMERCIAL LICENSING + +Commercial Use requires a separate written commercial license signed or otherwise expressly issued by appvoid. + +A commercial license may cover, among other activities, internal company use, production deployment, monetized products, hosted services, APIs, client work, commercial fine-tuning, commercial Derivative Models, and paid redistribution. + +The availability of commercial licensing does not obligate appvoid to offer a license to any person or entity. Commercial-license requests must be directed to appvoid through the contact method listed in the applicable model repository or appvoid’s official public profile. + +A commercial license issued by appvoid applies only to rights controlled by appvoid. It does not replace or override any Third-Party Terms. + +6. OUTPUTS + +Subject to applicable law and third-party rights, appvoid does not claim ownership of Outputs merely because the Model generated them. + +However: + +(a) using the Model to generate Outputs for a Commercial Use requires a commercial license; +(b) this License does not grant rights in third-party material that may appear in an Output; +(c) Outputs that contain or reproduce Model Materials remain subject to this License; and +(d) You are solely responsible for reviewing Outputs for accuracy, legality, safety, and infringement before using or distributing them. + +7. ACCEPTABLE USE + +You may not use the Model Materials, the Model, a Derivative Model, or Outputs: + +(a) in violation of applicable local, national, or international law or regulation; +(b) to exploit, harm, or attempt to exploit or harm minors or living beings; +(c) to generate or disseminate verifiably false information with the purpose of harming another person or entity; +(d) to defame, disparage, threaten, stalk, or harass another person; +(e) to remove, bypass, or misrepresent license, authorship, provenance, safety, or attribution information; or +(f) in any manner prohibited by applicable Third-Party Terms or an applicable acceptable-use policy. + +Where an applicable upstream acceptable-use policy is updated in accordance with its governing license, You remain responsible for complying with the version applicable to Your use. + +8. THIRD-PARTY AND UPSTREAM TERMS + +8.1 This License applies only to rights that appvoid owns or controls. The Model Materials may contain or derive from Third-Party Materials. Those materials remain governed by their own terms. + +8.2 For any model derived from Falcon-H1 or another Falcon model released by the Technology Innovation Institute (“TII”), the applicable TII Falcon License, Acceptable Use Policy, attribution requirements, and NOTICE materials remain in force in addition to this License. + +8.3 You must provide downstream recipients with complete copies of all applicable upstream licenses and notices, preserve required attributions, mark modified files, and comply with all upstream publication requirements. + +8.4 For Palmer-006 and any other Falcon-derived model identified by appvoid, public statements about the model must prominently include the following statement, unless TII has approved different wording: + +“Palmer-006 is built using Falcon LLM technology from the Technology Innovation Institute.” + +For another Falcon-derived model, replace “Palmer-006” with that model’s relevant name. + +8.5 Nothing in this License grants rights broader than an upstream license. You must comply with both this License and all applicable Third-Party Terms. If You cannot comply with all applicable terms, You may not use or Distribute the affected Model Materials or Derivative Model. + +9. ATTRIBUTION AND TRADEMARKS + +Any permitted public distribution or public description of the Model or a Derivative Model must provide reasonable attribution to appvoid and identify the original model repository where practical. + +Required Notice: +Copyright (c) 2026 appvoid. +Licensed under the Appvoid Noncommercial Model License 1.0. +Commercial use requires a separate license from appvoid. + +This License does not grant permission to use the names, usernames, logos, trade names, trademarks, or service marks of appvoid or any third party, except as reasonably necessary for attribution, provenance, compliance with upstream requirements, or accurate description of origin. + +10. TERMINATION AND CURE + +Your rights under this License terminate automatically if You: + +(a) engage in Commercial Use without a valid commercial license; +(b) violate the Acceptable Use requirements; +(c) distribute the Model Materials or a Derivative Model contrary to this License; or +(d) materially breach any other term of this License. + +For a first breach that is reasonably curable and does not involve deliberate Commercial Use or unlawful conduct, appvoid may reinstate Your rights if You fully cure the breach and its effects within thirty (30) days after receiving written notice. Reinstatement is not automatic unless appvoid confirms it in writing. + +Upon termination, You must stop using and distributing the affected Model Materials and Derivative Models and delete copies under Your control, except to the extent retention is required by law. Sections concerning ownership, Outputs, third-party rights, disclaimers, liability, and general terms survive termination. + +11. OWNERSHIP AND RESERVATION OF RIGHTS + +The Model Materials are licensed, not sold. appvoid and applicable third parties retain all rights, title, and interest they may hold in the Model Materials and their respective contributions. + +All rights not expressly granted by this License are reserved. No implied license, estoppel, or waiver is created. + +12. DISCLAIMER OF WARRANTIES + +TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE MODEL MATERIALS, MODEL, DERIVATIVE MODELS, AND ANY RELATED DOCUMENTATION ARE PROVIDED “AS IS” AND “AS AVAILABLE,” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, INCLUDING WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, ACCURACY, RELIABILITY, SECURITY, OR UNINTERRUPTED OPERATION. + +YOU ARE RESPONSIBLE FOR DETERMINING WHETHER THE MODEL IS APPROPRIATE FOR YOUR USE AND FOR IMPLEMENTING APPROPRIATE TESTING, HUMAN REVIEW, SECURITY, PRIVACY, AND SAFETY CONTROLS. + +13. LIMITATION OF LIABILITY + +TO THE MAXIMUM EXTENT PERMITTED BY LAW, APPVOID, ITS CONTRIBUTORS, AND ITS LICENSORS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, CONSEQUENTIAL, OR PUNITIVE DAMAGES, OR FOR ANY LOSS OF PROFITS, REVENUE, DATA, GOODWILL, BUSINESS OPPORTUNITY, OR USE, ARISING FROM OR RELATED TO THE MODEL MATERIALS, MODEL, DERIVATIVE MODELS, OUTPUTS, OR THIS LICENSE, UNDER ANY THEORY OF LIABILITY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +WHERE APPLICABLE LAW DOES NOT ALLOW A COMPLETE EXCLUSION OF LIABILITY, THE TOTAL AGGREGATE LIABILITY OF APPVOID ARISING FROM OR RELATED TO THIS PUBLIC LICENSE WILL NOT EXCEED US$100. + +Nothing in this License excludes liability that cannot legally be excluded. + +14. GENERAL TERMS + +14.1 Entire Public License. This License is the entire public noncommercial license granted by appvoid for the Model Materials. A separate commercial agreement may provide different rights and obligations. + +14.2 Severability. If a provision is held unenforceable, it will be modified only to the minimum extent necessary to make it enforceable, and the remaining provisions will continue in effect. + +14.3 No Waiver. Failure to enforce a provision is not a waiver of the right to enforce it later. + +14.4 No Assignment. You may not assign or transfer this License or Your rights under it without appvoid’s prior written permission. A change of control, merger, or transfer of substantially all relevant assets is treated as an assignment. + +14.5 No Conflicting Interpretation. Headings and examples are for convenience and do not limit the operative terms. “Including” means “including without limitation.” + +14.6 Language. The English version of this License controls over any translation. + +END OF APPVOID NONCOMMERCIAL MODEL LICENSE 1.0 diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..7b3dc65 --- /dev/null +++ b/NOTICE @@ -0,0 +1,22 @@ +NOTICE + +Palmer-006 +Copyright (c) 2026 appvoid. + +Palmer-006 is built using artificial intelligence technology from the +Technology Innovation Institute. + +Palmer-006 is a derivative of: + + tiiuae/Falcon-H1-Tiny-90M-Base + Developed by the Technology Innovation Institute (TII) + Original model license: Falcon-LLM License + Original model repository: + https://huggingface.co/tiiuae/Falcon-H1-Tiny-90M-Base + +Palmer-006 contains modifications and/or additional training performed by +appvoid. The modified model is distributed under the Appvoid Noncommercial +Model License 1.0, together with and subject to the applicable TII Falcon +License and Falcon Acceptable Use Policy. + +This NOTICE is informational and does not modify either license. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b637edf --- /dev/null +++ b/README.md @@ -0,0 +1,494 @@ +--- +library_name: transformers +pipeline_tag: text-generation +language: +- en +license: other +license_name: appvoid-noncommercial-model-license-1.0 +license_link: LICENSE +tags: +- palmer +- text-editing +- rewriting +- paraphrasing +- grammar-correction +- edge +- small-language-model +widget: +- text: Make this more concise. + output: + url: images/4.png +- text: Make this friendlier. + output: + url: images/1.png +- text: Improve clarity without changing the meaning. + output: + url: images/2.png +- text: Improve clarity without changing the meaning. + output: + url: images/3.png +extra_gated_prompt: >- + You agree to use this model in good faith, respect the LICENSE in this + repository, comply with the applicable Falcon upstream terms, and not use the + model to harm any person or group. Commercial use requires a separate license. +new_version: appvoid/palmer-007 +--- + + + + +
+ +
+
+ Palmer 006 logo +
+
+ +# a small model is all you need + +We celebrate three years of experiments with the release of `palmer-006`, our flagship and extremely small language model for local experimentation, edge hardware, small SBCs, and low-power devices. Note that this model serves primarily as a language model and has not been validated nor finetuned as a general-purpose chat assistant. Thus further finetuning is required to properly work as an instruction-tuned model. + +## updates + +- [🐦‍🔥 A wild palmer appeared, go catch it!](https://huggingface.co/CEAMFA/palmer-007-preview-GGUF) +- No longer promoting arithmark, after doing proper evals, palmer-006 is barely better than the base model used to train. Model is [#5 on open slm leaderboard](https://huggingface.co/spaces/AxiomicLabs/Open_SLM_Leaderboard) +- Second spot on [BananaMindBench Leaderboard](https://huggingface.co/spaces/BananaMind/BananaMindBench-Leaderboard)!!🎉🎉 + +## ● details + +Palmer 006 benchmark comparison + +We first lightly fine-tuned the base model on a diverse set of highly curated data points. We then “palmerized” it by merging models, followed by another light fine-tuning round. Finally, we adjusted Mamba for maximum token speed. With only 90M parameters, this turned the model into a competitive baseline against models above 125M parameters. + +> As seen in the benchmark, we are getting closer to the performance of bigger models with our current technique, even on models of this size. Note: We used `acc_norm` whenever available. + +## ● quick test + +You can load `model.gguf` model into a llama-server, then run the script inside this repo as follows: + +```python +python completion.py prompt.txt +``` + +## ● license + +Open research, education, hobby use, modification, and redistribution are permitted. Commercial deployment, internal business use, paid products, hosted APIs, and client work require a separate commercial license from appvoid. For commercial licensing: [nosoyhackercodigo@gmail.com](mailto:nosoyhackercodigo@gmail.com) + + 👋 But hey! If you already have a [donation subscription](https://ko-fi.com/appvoid), you can claim access to a free commercial license through the email above. You will keep the rights as long as the subscription is active. + +## ● upstream attribution + +**palmer-006 is built using artificial intelligence technology from the Technology Innovation Institute.** + +## ● fun facts + +- the new logo represents palmer's eye, pointing how small this new model is +- palmer models still exists to some extent due to inspiration from Karpathy's idea of a self-contained "cognitive core". +- we just like light finetuning because we prefer starting "real" experiments with the strongest baseline possible +- [we have an org with early access checkpoints](https://huggingface.co/CEAMFA) +
+ \ No newline at end of file diff --git a/completion.py b/completion.py new file mode 100644 index 0000000..9df6693 --- /dev/null +++ b/completion.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +# quick-start for completion models: +# 1. start a server: ``llama-server -m model.gguf`` +# 2. create a text file with your prompt like `prompt.txt` +# 3. run a completion: ``python completion.py prompt.txt`` + +import argparse +import sys +from pathlib import Path + +import requests + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Send a text file as a raw prompt to llama.cpp server." + ) + + parser.add_argument( + "prompt_file", + type=Path, + help="UTF-8 text file containing the raw prompt", + ) + parser.add_argument( + "-o", + "--output", + type=Path, + help="Write the generated continuation to this file", + ) + parser.add_argument( + "--server", + default="http://localhost:8080", + help="llama.cpp server URL", + ) + parser.add_argument( + "-n", + "--n-predict", + type=int, + default=32, + help="Maximum generated tokens", + ) + parser.add_argument( + "--temperature", + type=float, + default=0.0, + ) + parser.add_argument( + "--top-k", + type=int, + default=1, + ) + parser.add_argument( + "--top-p", + type=float, + default=1.0, + ) + parser.add_argument( + "--seed", + type=int, + default=42, + ) + parser.add_argument( + "--include-prompt", + action="store_true", + help="Include the original prompt before the completion", + ) + + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + try: + prompt = args.prompt_file.read_text(encoding="utf-8") + except OSError as exc: + print(f"Could not read prompt file: {exc}", file=sys.stderr) + return 1 + + try: + response = requests.post( + f"{args.server.rstrip('/')}/completion", + json={ + "prompt": prompt, + "n_predict": args.n_predict, + "temperature": args.temperature, + "top_k": args.top_k, + "top_p": args.top_p, + "seed": args.seed, + "repeat_penalty": 1.0, + "stream": False, + }, + timeout=3600, + ) + response.raise_for_status() + except requests.RequestException as exc: + print(f"llama.cpp request failed: {exc}", file=sys.stderr) + return 1 + + result = response.json() + completion = result.get("content") + + if completion is None: + print(f"Unexpected response: {result}", file=sys.stderr) + return 1 + + text = prompt + completion if args.include_prompt else completion + + if args.output: + args.output.write_text(text, encoding="utf-8") + else: + sys.stdout.write(text) + sys.stdout.flush() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/config.json b/config.json new file mode 100644 index 0000000..33b01b0 --- /dev/null +++ b/config.json @@ -0,0 +1,70 @@ +{ + "architectures": [ + "FalconH1ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attention_in_multiplier": 1.0, + "attention_out_multiplier": 1.0, + "attn_layer_indices": null, + "bos_token_id": 1, + "dtype": "bfloat16", + "embedding_multiplier": 0.11083984375, + "eos_token_id": 11, + "expansion_factor": 1.5, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 512, + "initializer_range": 0.02, + "intermediate_size": 768, + "key_multiplier": 1.0, + "lm_head_multiplier": 0.078125, + "mamba_chunk_size": 32, + "mamba_conv_bias": true, + "mamba_d_conv": 4, + "mamba_d_head": 32, + "mamba_d_ssm": 768, + "mamba_d_state": 64, + "mamba_expand": 2, + "mamba_n_groups": 1, + "mamba_n_heads": 24, + "mamba_norm_before_gate": false, + "mamba_proj_bias": false, + "mamba_rms_norm": false, + "mamba_use_mlp": true, + "max_position_embeddings": 262144, + "mlp_bias": false, + "mlp_expansion_factor": 8, + "mlp_multipliers": [ + 1.0, + 1.0 + ], + "model_type": "falcon_h1", + "num_attention_heads": 8, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "num_logits_to_keep": 1, + "pad_token_id": 0, + "projectors_bias": false, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 100000000000.0, + "sliding_window": null, + "ssm_in_multiplier": 1.0, + "ssm_multipliers": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ssm_out_multiplier": 1.0, + "tie_word_embeddings": true, + "time_step_floor": 0.0001, + "time_step_max": 0.1, + "time_step_min": 0.001, + "time_step_rank": "auto", + "transformers_version": "4.57.0", + "use_cache": false, + "vocab_size": 32768 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..c4fa8f5 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 11, + "pad_token_id": 0, + "transformers_version": "4.57.0" +} diff --git a/images/006.png b/images/006.png new file mode 100644 index 0000000..10ef5d8 --- /dev/null +++ b/images/006.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c353c1a1ac2b5d9768f5f5ed9b54f82c784fb070b056b6f8de3d1067b91bdba +size 135721 diff --git a/images/1.png b/images/1.png new file mode 100644 index 0000000000000000000000000000000000000000..a5c395e3d3dfc4638645284a66a690636056c68c GIT binary patch literal 8565 zcmeHMc|4SFyMJaRTcHJIYbsgFmIfKy*hjL2q_Pb&b~Bn8+mOam(nb_TmZXSKKP1_; zp~P6TtL!5C9_JpM-e2!||2Us>&VT3a^U?F%_jBFXec#vhz19ggG1A-4&d&}(&~|-{ zwkZU`!7ChMWjy@(wX(p2%@cE$3_+Z48Go?5M>qu_h*`_U+>&BxcoOA6^pM8li1v7C zUk^_}4M8gEzMfbIH#`Mlk9Tq*sEYk8DicGv;8exV$Q#NUdTQaFT`*J<-i&Hw?m%^O zP{N6+tFf#2q5uOAJOzvJ^>8PUQNF5T>wHn*oiQyV28c*FN0h0y?j{KssfsyMD4r-8 z86O`XX`f@#M3R%tQ6(iM8Cf|QIXNjnAw~8hP_VvI1oA<^fgQ2VLmN+aAh~!_T!;h& zgD2LW=tWT#6JtkgveuwbNG|rjxf96J*!3AN#u908X&EezEOS&^mT@cuLS=&pOX3No2gapC?{b%oI;1dXXIP>l_%1HkoLV@K_2SrzR&WrywP( zC?%($BJ)q57<>Fuvp(sppO5UugyhD8)h0cnY~OfARPE)=%>1;dFc{`ZzTQIUH62 zcT7pj(OwQGg>-PhN?{cp?WK+>VX;S#DLNqK6_tK-*;w)?8(n+AMP5!`PC;JjDDtSH zlKgSm-wW11{z-36^m1@!$XQM9H`zbt)@fB_7^-)7+0fYfU{htkS8V8WgYa*d`S0Th z=kOato?ayPb$sF+Wbp2I4?KawK+w@m1mPS|6c>s+{vY_!aHsrJ!!aS_kI*&4$d)=i#a2jBy^#j(v>fDuZ=i{eZq zsh!0V-0<#x2n}~{ED29lVNm|(H(-YA^8O#m6!?kXLjB_?zF5z{nIs@Xt(y{&WKJZy zt7%~g-dKdVDV~7Clkg-71TYoVznKSuVG^o;GXumr^i}@`cNLj`X8JELvT--yr+~+j z`R&)h;J0T4XHXR*0l#M-FRlfArV-v!-_g(zIt<=fp)D|eh#9=Wz@HjS;P<;OOdMib zp9k(4zH9THLACME4m=p6^`{SW&maT|1nO&RnEO8X(OKpe?FW6B6l;CR6Dn?r%tKV1 zS_=w(eB5h-W#sUI_fLBf;zJ0|c57!16Yebq*}R%z4JRsUD>NpYhD@HaSK_%@P7#lf zYrmQvFK#Ec*Nw{K0uqg51`}l1o}PxZGwupQ=)G`=`ve4OvO#EZ7{q!Af*x*x5O*Lb zbSFeJ`S-{F-$I}nX40LJ@wJY*h5qy7LdxDvqq80zFMlnb%V}j{Vad#(P7l=P^o)2H zWoBk37^O817Y8g(y$R!b+{3=I8(uE@!mTy?kzT}jM?ubFSWr-qeBo3_0r65qMXpmF zFCX8}p0YC0*vsFx(6&D-E4$?G?jErI?`L<(D7lePpscRmLn!DNo69_t(=zyE;utF= zU|eFOe&O5O+XwaDzklB{;;w|Cdi7Z9XM`t?-&ruu%gT4^D(+yg^GOSR^W4iOFPoI7Oq0B_ePfvdTFg!dAZth*w)o{L70FLHT@}21ID)K2H z_SjIX+Rs~Cs|GAg0C5{|1?2=vs6rVT8B5E{X-}RwIyou!He4iyv+Yb$}apc6JWdB^lJg#Gn&y`R5{pRnHZu zlf()3_Vz9=PwCEggyE%3$>~6&)};Z3GuPl278bg?y5Rp4T>Im8LGP?yQzWK`ZBI8( zOiVO1G)zo*4)50{!;^222R~bv1(wWQtBi?_)zi~UU=r7X#O>_uzrDXH@c7J3Xn%kI z?c29ob8KUUH4nh-O7hQT0hq+v(6wxeeUGA08L($%J-|z@t^MfoH!;WWW0`nU`BM7z zg|j;K6E!Cu1=k#6GSk?9?1US?SzLug<9&icKT_VT7TLV-mV%L^l38Qwle30TCCqL; z4i>#1E}nvTtK;6E!paIS`s;b|rG;BLGA}CC{O1DZ1xu64yvoY~Ei&cB#aUIW)?8d%lmb_LZEU)2Fa7eG9cg~@IbVx!(KWc8gYx%SC&pUVSuy zyUurMhOWnZU1#oTXLEPA@ZP;`ot-y?L-Ve*LKw&U_wScJUy8kRCp9%yWw9^XKC?_j ze|c^!9_iA+AyZu6&~V_V`#3*}GGriA5}FW%3|EGt+JGTCI5@mG2F|dLgICg-NQ~`o zw0CeIlSrQ&b0w2R2S%DRfng*hB$V&P_`hO8C>;_JDL=E;6S(ZLy0S>hT8$Fn;o&)S z=umS@OU-N$yt1;=BHM~hkcJFrMw8CHbl>YuTUlB8Anbo~uE^=*$B$-aX6MhJ@Bd|4 zP`onrv7FK%A`H9j4oF5uN8eOdnlM>rwJa%x)UG2yfm8Q|8sgdKVXq$hjQWKpy8tL-)B2{X*8=K zv~1#-XHwwO@anJSl7w$H$LdtxIJrgN@ZF+4H8DR=barNCV+%@+jfpWzK;;(|oez3e zRaG@LMGC1%Mz@sVaX3ieD)I{0T8~Oaxz5IlxB&lK^9;%&T`CV^;^31umjfZJrKt(cUPma1*mE=`E6Z?kgK%qLVBmFLDb2@6);jmc z*S>UV6uqoP+$)ExYBW~g9xgiem6L{(b$EZHG0j*j<@&X2PYmOq1Q0M7jG^I8y0f|3 z(h&WTgoH%RVALGIx*Sxx=mxAWvU6Zy07N6rl)=vl0{r~!;i5OOu*hu#bxB;+UdIL1 zR~HV7oQp~Yk@R5-P{=8I>LE#J${dX@f1pH0(A(PD%0E6_TwDaIjU39$Lt>;|wsFu@ z`11P?Fu13gTt5Xg>+P)|vh*@R*+-Kk#cSZ}dt4+g zFO(UbsA&yboo%%h;$#7zuU%tjPE@ChT_#>c7}~=k^|ozg8j%_SHF#vF4W|%lOLadU zni2$CM%}x&hnpJ;7w%(*Rz{>;wt_i$A)7%|Du_!lF)^Ac{?)a$VH?S|)mUcX z3+dBI*ZE{#=H;PNmMP<%N>xH^6%`dg3ubB?9DP^G*Bghm6X26_kY6y^!bF)iJy@qj z`WbAy<$yf6{_!Tx9L762rSq%os|-`^7As4npye`NLPWo-tLyIFyOBd6ylav4nE+STe5O~F;hiD{w$07WkygAAn*-S19M;s* zqDfl5%zr}XlLkQ+UwUS457@V7Pv0P%nojD+G{qNGR;QR z=)ut7AWOJtK45kA!uR(Nd4-aspeDO=x%Da5CdN@>Wo&hIdVs(Yle@2K13_;YIb&4R z)lH|v!pF~Cyr_DZ+LV4O-Pu&07YYZrz0zk>7AO>@%x#pm%j^1IZSGh4qT0bSZ48E0 zq&1cU9hEvT;E0jtpuK$#nE5~#fplO0*H>3p*WkXtAPfMtcCt5=Zu^4esxoBC2X;a8 z(6~z3dW{qbowUBG1X+D$!9|1GVb7tZ-GYM88YkD}3U8_- z`xMmG%aW53n7mrYkUmasSCs^#unZjIf}L zTm(F`hR4Tsb-1>8ssNP%#B|Wowo5{1fJOlx2;Vu&d3t?8u!Dnus?dAxI|X%$&IA*T zP=y-W;WJ|1T)u*nZrQ#geK{a z0THLXc(EhOP3}ITkSUZuVo-epC`XJKQ1qc`YH95+tgWq$v@%r&sIr62u$_iFVUc?_ z2lB$380APdR)HVz0uGdw7oGIU@t8;~(9atwR6-NegF9_&Z6dHwozwpAf~!(0)(Xqj{&G#-Bl_(X9NyjfBu)hF) zuXe!yXUUnx&LXCkogB1KV7i@)(}Vj9>2!KzXAyL3(~tNp&A4E)xf0=qaY(i(x7;?6 z8ScX%T_AAm1SRs;sL>E+T03kvFYl&bSjnU@B|0-`XTQ`7_4!>h_l&9ug+ z0l>XKn0n~*uyqtw54f|=LC z1QwuoHyR`-BJ=Y-m%;=~Z!?0RnV4~+yPKPtnxd*|moxHkl7AY@z5rQC$!$9mqNACr z4@ofs_X8s_+)+9Hn;;W0Rhu{Z7%QWvlM7XmK_&|ukt_cT{}EAGFf_aL>C;Kf%G|`S0ogV=bu~4B zdkmj#zz&|;>3vbm9JD@=SZ=Sr`2D6L1yvq#0R-U*M_mXbn;zcK;Y0Ov8OkC|z6i2a?_PBkW$DQ_UR-P<4iBD=H}H>ga$n z(K_fEFdKMmFDPcicZsDm=sN(9+Xk@mDACwOA-*;1d>R^L2=xQC@jR>~JScoX8oTGx z43sJ%8a@0qY~K~_4<9}xsrrMYpviv+DnVLV*?*+zY4`P^){N7caq;m%Op!4YtXKPx zq`oQ;LYo#})jGz@yT!~CZg)M$z5(S!@yu7I3nNVuO`~G^(I8DFy(aauk6eNDqi*Pa zmnmD3vwzD<3|Ii#q*?zuWV3px5Y#Vn1}9IxRZA;PNwNQ0c^M>cq>IT(Nj*U1FI<}s z7pj?@baHY6DH$+hU0?*CrlzMO_Nj9UA`TzUNKUpJ?=1W)AkHXld9H)_l~F%{MneCH z@zggRyC_=+hl8Mm*&4F{$dMz!3ChUiH3;s>Z6byQ%CA z{JfCgyK6j0N|#2C%Ta42c9hO_Tr6wJv5gQ8$^(_s6z?Ot zrG*9W-o4?%+Fco$nPMmJmCs+gtvKP9+$W_F?*Ym~P^sBph(7}S}WLo{I?y!VXLxKOZ^-YXBw=&ac4Ekp}#v1J(+@RD*kQ*D{lf|~@xeRFp zSmW{i!OR>gepiI(&?G-3;Zzm5pP88%^lWswz7?TRsBhoDmn_*Eq;vov*1YH#12L%d z!uPFI?jDdxzH3g}4J5B~+T0w6$3IM)I2Etp(VpwXMP=vL_T5gq1Nt*9-QAy+nW##j zZPxJEV6lnC6nKB=);{&_=H_={f}Iw?LC3^+$NKPa`vHF^drL_9N#k%Im7J5o!^Z~@ z6904gQ2z^vbpSLw-dx#kU}U7r6>M&4c`quevN;K1358N7dMheAVZgD1W{-!93#0w9 zn}b%)DXj9Pp&=?J=1JrB?4PM!=YD}Ag&tDh`%a*uO6*gjDv(RpLERf@u9V1qfeT*? z2NS=awyos1DkI;nLwW!BlIPOM&dy#R;%lv^_W__YM%ucjIjI+jq7+nepoH?kVzF#& zY(r;@A3u90DK9?@8e2meS4BVptoFF5Xn8Q{Qg%j0O;y#zd}}+`AZT~+N}Z$h*YM0B zwAbDpL`*3ud;hqn;`difVpssMTTt(VCEbLAM^5{flGfI{PGg-VCk}EzSFj!;kq0We zHJ_c_{Z5y|?0vWHGxseT+{GNk{pr1@zr3}>N?7_bts)eHG4NoE?-K5uMIWXA%FJ#1 z>30pAaX-i3$!-sPWWitLB!%9j^$H&G%D@aIKm{;f2oZtd1DM)S$qX*j3;_dAXx~U(#r`*7A zs>{pEpf?*45rHQLCGCK&RD-ZlMxB&(3^(Sb#oW*+N3H z_%+ZOu*vZuk)~FCMstdlS9ZY5%aTo>=AFv|T}#(|(1*-y9ueyr8W;c>wpUSCW22e+ z1&1$e&{}eMcsOX}I3xLQ>gKjRNKDLbcHQ^767(p>#>Q%WmVT{@d0fd6gS68vGH0nJ zHaV@X`AciNb?86mSwbr8WPR2B)Ya70V^#O6CH(v6ihuuR^MCZi)fz0z)0{j#hRFt& O59#X|X=iKN1^)+qNvJje literal 0 HcmV?d00001 diff --git a/images/2.png b/images/2.png new file mode 100644 index 0000000000000000000000000000000000000000..9bf029227bf9580398a2d928ca04cd45e9ca83f8 GIT binary patch literal 12515 zcmeHubyQVdxA#U65hTPwkT@WUqJ(s}G=hL2DBX3aLrX}AG=hSFAfbRDAV?n4LIIH$ zq@}w%?*x6H_j&Gj$2acyzJK362A&;juf5h>bN=R>zqt%jxhX?TNJWSs2(g^3lq!N8 zfzu-h!NHFQm1GY59JiCzazqf4PX~WEPp^`kN06hE7V38}ca#(bjBIUK4UBCKO;}xR z?4UJ*2#LDd85miaU{Ho8W)^5+rup(}CX|J-Fq0;?61$R}q=~tOth}5wXsG!3b+b09rh~#=Lfghn4pn^v8jNnl=Pn_a3#!Sj=|Uo zu(7$gxUjl#vf4VBv0dlq=VxQ*VB_Fmffg)|ZfJ~wD+}830`x$LI_yKr#L>vX!VY6$ zi$)#vX<%sUgb`+9B1HY^S{#FMurU0qceEp`!QmaJgI8E#(rgCCj%?Ri*$;+=LJ9qD zFCcF0cyRgqBWfo0hZn!EtSt_|WM|;uXrk_BXClm`YT{_?#zLFfqWG z7>jVQbMUaR^RjU82(kUQM+e{d_X|=67y}WHOZHB-7(ruOqu(b2zkj5}WpA0wIcT}L zu>Zbs?e~w9qlv>m8r~L{wzIZz;Bc1V`$t1V2LrV6KUzv0v~=cI=e@=C`@ZL2<2xM5 zKL>Zi6(eUX!pYCY!N$jHuIs$~+R&OpY7Fcos7&6+V=8~%7GnEb$NwJ_vHP2? z9azJm$sgDOJb+nn{+R__35Yvk%xxV+v<%QzCf05!acgG-2NQRpgO>k#24;9T;{PX^ z0#5uT)PD@c)xhp=CJD_14o%6{LEYBYT13(S?QDR$tZITbHgPa&YjE)wGQt>ynH&K36n*;G05flz+>tX? zQbL&FoB+YYp+b(r84mo3;GF;KTpH&xf_r!$;Q7d@Kj#Onejobqb8vO|>vD8p96^|8 zX?7ur~9&K)z#EHa>+;`pGE3q2DPJJKE06s?2R(n}|3pJp zA&5^!o`gL$?Aj~pTiMr3Zq@cGlGiux>+kM)?@Dgx4~X>iM|^otk~cp+<+1AN?3rh! zse$hkS)fXapono00(k`SB}aUc@etJI|6Kk@8vj>iV4xCow!U?Eh>O3^yp<*$dhYuD zjfMIBv9U3)osIlmLEC{U8(UjjD=Ux9rBT0X&y5B8RW9NqOP)UupL28DEVO77U_BMh zq&$};vo}9F{5?^Sn3%XTPj5a;X3VTMlu}JYgU6^gG_{(TiYrrrwha0kwk|F#yjSU7 zl#-I7KpT^!h-{@lfBw9+wH3bE^g`6T{>ha-$7WvJ{(DvTH-|olP_XIVUz^cdr{?mV znVE@-iVA@~;U$uAj_VImLIXSu8w>w&2DYWUX?>Cj$ z4zTJ~Re103VX;^c^{XUF3NoMmoaE%>tSrmMc<$jZ5z3XLn(^`RB-DI%I~(>d&hJZj zd3l)}9HTGgqE&4-pKhH6?I)01c|EI{^C6@HqUAN zHIn6Sv2}OhYneW}iHh{}ba-L2UL~-4Zz~&%b)Wu`^D&SV2IEY6rt_1HY3;Pey zBb9R1Ub{bXb;>=r*LHSx+VAeQPqt;i3hitx4!282)z#N)=No>8284u!Fb}oW)z$O7 zXlpe3rm(-aG~ym+KU|wA;AA=WE$($7CQkWjtzbqk_#M-gb$2=A6u~qa6L@3U^pf0VUCy!V z*YsC=0o#(NZ752Y5`NAEk=5U&72VMD=z{|E8>8&0DL}YI(AQFxbzZKZ6B<2w%Y$K$Y6BlAGJuu&}aLxy)R2h`uGynN;N*?v;1nPXxJV~7p2Z& z$`u_)#q?BBiMjT*s?0EXnlue3dNeqg{3wMKD(o6VIYA?{FW1WUgVT9!M1?EMz5A7Y z&ieEKLwrY>F#b05K7F<%mjdGow5Z(MKLg> zzJ7gNhhURf_!#Q=-rin@@{5@E?464rdU|>q8WeQrsg_q(D(rswW@lf;lGQCPED(^A z4!koX#gZ+Zr$SuiUo8R9kb9?yHkaBDgWjSCs=`=5F%Sj}O|+yL7#K`QFf#M;m6nz9 zb90~3AvlSLC6Xnq+F5E?Sg^yKi7E!g2kcJCe*5(cowv0@p6jr|$A`pf>5Af-;&=~o zacUZ$^Do2bCzy?sOG{k=tHIPJB(Rbhdn_+IEM>x|04~B=KlQgDV(@va<9K*^xgeIz zjSp4{@UG?dv(cfUsDuPT%!CSte#vfi>h@jtk}|~>ZsetH)#fE;WwiMXMskOoH+3&GgFMh z@G=S|me%rhBBV;Sxxb(G>{;o7lenqTPtOqs81@v~OfZkgyi%20)+)=nThcnUUr0?u z1D^=a{gB&&)VQwqxr{p84Qa1fs=ul#Xv^chcOSc);gXt~Ix})ttW_$*2kGtYo$Ifh z9vgeeDwWGcrF7@cGbs@siB7K^^;@@2o;%mo+Z)RIiMHj2zE9YBUXlHsWsujmhKB5v z6mbSkKT){B?>Ls69eG_-Qxk@w=|?|7AZ86%!FKs_GOv`j9JK1|>jOO5UYYb`k(>ie zl8}@HH`x-T>f({r=VLNS)8?5VF zgO6F+*$=qYZ{L=i{m7lj^o3Oii&>rO03Tsh)ZN;u`c%H-r;@GhJgD%*g!%ERP$fPJm|$4(ecBUW=b8CYZg_I;R09SNmKBXoTI>^E%fz2G-{Xynhy$%vbMwiHZ3T zjA{+Bd2cqDz3_Y}@4Y=+0?-~R&l?jJRRG2U05XZe((W5kP{e}W-_F(uqvluj@4lAr zwmjz2?{@6Z2kYzW#r^P8vdQqoyTCJJou=n{%i2P!JvT>VtGA}Eii(Pk-4m-Oh ziMQWXk}!K86z7ssJh<@G{+vBu6gbqHfdTK)`lkzQ9e5$QAY&x2ACtqUZ6ZoS?&&^R6E*SlIqXp5Marw`op`q7m89#pfDCnC} zOL-?|n5rgbXotZ-V286%x$iRHANJ^xxw-jZ;Vvbo>2p5&BOCWWK00~f)BVs)&?wid z>NjrOFg9ka4jcqK1Q9)$FAfP0ufj}{Gxy~-pY%C5G(7wu+gnodL#rIAQuiGZ@3!<$<;?tGr5l#geOB$R{7+{dcB~ z@wx*GMrFk3c_93Ya|z~$I4hOLTfe*vd|7DdiT*2~TYHlkFI+R8y#qnGmlX3&oF;RT zf2Eq;k=0=->+bHp1yRE1IznRN#`<~(%%=(i_Dh$>J_Ha|sGTOfVQ-(Gm&Z1biY6o? zl9ZH;)BAe8uBS&(R8;SkYEYz!^{)>BHOGxbCkTAX%N)m3HM}cztF3=>vci4#!)LACqvBwPcS_KSd|I`iCNHI2K;}qjh4=uy)K?nV zs;`-*W@lT0CS_-5KTAmj-;ICbTytyd2W}IHwB{?|s@A;7tJUQ>()y{^7hR=%kA-)d`JT_u=48mfF z#p=I*AB!&lR{|2V4`UzqBNh4D*m#CBKP^o%Mdj0%FCmoOb?@J&ym@ntg+*z{yd3oZ z{tX!!y$aWpQzP3c(J37}_enC3)?MR}hH4f<{ zgnT^cUcN=gSwbD$!KufA71;9-+YLofX2s$UHrlqBA+E8~iuC1*J zd#s729u@B@b7W$6-&&c>G#3=d^|@eXHZB)^Nj#O5Mu4$DkeVPeVvtK}GYbCI#Nk<;wEn8PFoqREB0 z#_`!ROs#%py7}B^gi4HsQfQ45Jb!@uUc>ieHIS20Q=7*P1?-u)9VaCX92pxQHxg~> z7=;8Wve(B485_`*c3S@RF_0JC>mzkuR!M1eH=a1U-O>sC=tkM$aA2wsphQ_IW_JWH zP+3OThSM}S`(>ijU%nK}t)jBBu#i($4wM3nlyI1Z)zf8QU_g~+hB#eH=^y>DGezX2 zMuyjpi0MEe$c2=uFh9*yuF;MI^4BdiB6Yvu|F2%!5b zu6X707w0$eS$DCwU(k=DLNd`00|VQf^lwhQcuLfMWf+2b08{Z)C9LzTwv0^RMj1FL zlTza}td^dho~r7RAf79tqSaMZx;yut%HN3avzNg2ArdfZew~vu`y=OW@t z&Pr%4OB(phLoz8#L517{pYg7s09Dpk>bcFs82c;8Xv#G9+c!m7Rvr97e|&@@lP0F3 zLP!?PtK#C@U4tZ}ol|gXjuRtm6_u5@na?n4`zqjPF=}vj)zw*idUR5eXSI;AOQ{`6 zRug;PjOaHtHIbgtQBm;;<~demys^DqURrvmIx$5uFjaR_95*>c{?(B^Ta%CG$lJiJ zSN-KKc|}EM$osnVp}v-IZ49vxZ>*>Qz4}m$Wx;POP3cYE?4y!ACqGMD!bx zw+p;Q<_{RZwB#VWvheF;&HzDHLS|mm`ei1jV^Iw+6Fwm@S|1rCvX?eR6U7 z;~Cq2e<|(aN5(i-%66q_V|R&&a|e^QtP{(5do~qkK|M3!5_A%dj~HcMOoK!t$||L{ zR$4YUHa3>q`(_`PTzVg4tvSN}P@5w@A%VW^EM)j688y>o5c2s;Swf_wq_Vk(aZ~NLdpCz3m#VPIkqE_uDlOMTTGB* z&@Q}Q;l~>yX^OEYp2~lIOWq(?kx;V)_iy0(Tm%mooPO~mb&+;ox#&xD{Dm)Qa`?{M z_6%h@b*@A%_KePHKBqQne&^_CQqaEJjO*AP-MmiwsdRpanM>D$@DYU?y>Z|=ZneZ9 zF-%_$RE8g{{46mN<5T?|@KC+*qR;3m_DZ{w<4Bj1A^E|B?a)HwXBU1q=0dsvsL5wJ z<8iMkhj${G31iLNzX!hKlqynYLeD?Psmj}k6C)#oSc5!+)+iev4@Zgjamj=aKR+Cn zHpjPr-RWaIka-a6d6>&ZKdePKb-CPuCy4UHof%Ne>r6{!Noku@X}NjxX63fz-=+a< zP^Nj%e#8TJxh(z9bIdU7LGvdY`HyG7n2(nl^9cx~rn2B%%j=BDf#Ezh;ti7HLhc;d z6R@cJpTy*P(=Qx(7X7DLs;9K$NNJPo!JvN+@%Y9xo^I?rf9fVGwLjfO?IqTcfY4>a znTSCQU-J__F~TCZ$l*GUxE>hkza~Sz(0OICgi%0HFfDb|6;G^>D=_@G7#BJxhlYHE zi}BtH@zdg{@HRqRa;OA5PV)EbR9>rRn*ZZ|dwaY8fd)L+%^L<1iG_v5IwYSA<}PqV zKLi#R*u6}Ofwm}vaBelN><*4MpB!KzeYWrLz|tE*&KP+wh>fyl>5bMW3*_0~t)gAT z$kVfuJ5M7W9gin;Br_{Z;zkclC$uHXfZVd~Z2=7!jY=PImMNdApP&Q}%r^0egOgLh zh&I9Pq^DOw?-=8pqYrIrig#*mj-;+A62cxv2q}NWi)Q@O2-~CR+tkFMONit6|9mA& zhzb$Pw;<;u!R`ZD1t z+uwKuY1hz@p=_b?F`tMCQfBPix6Sn|$t07R2?sDCgX<&Q9wojVieIQbns*b-_QO^yq;D9J(zBqewZVmcA(sH@XBpj?3S#|~UZOlVDwm@Im| z-I9BXAJHQ90SW~?f=a+4LLwd)FJ6q4!2#}TY-Z+~7kV9N%AjD=0%I;-UdX%eRwrhR z&di*s_|;17?d{!e3u7XMFSbo$+}zwECxFBib0|MaPVO6QT4cPlHJzteyj;lWN~Vw7 z$EXY@{26k4R2jJt(P?Vx6LmRxd1kA^Y*VHb>b!yi0!z!w^ex3<;I|8nG2*kF=@}XC zx=%a3Bg#t9($WIsjx_I)hPa?TYH{XWlY~S0b<6h4vU8BuY1ss0e}2Zt$|V89g#hO( z>F~m1j$byHpOuvrIP1s>$U-y{>g(zr%y`M?B#@Ah4A+MFNpa3xasxDDg`5yoLK;ph ztgXrxki>3qggO3~h>?uUu@RL1*GPIld}4CZ&gad|D&Q(*xoB|gByfEwzDE5T9AtN2 zF<*3S6rW`;E-D%w8{_CjOL@G7UDBDJplT?+8n^FGy(lNO8mD)enP0a`um_-h&@Rb)&rLR0ET04WcrNuLA?3K=th0GXlO+=EgqIMb^BTZIcj!H#sN7sAPA5BcKrNydxn2 zLU7}igE)&i5#ryd^Ar@v>bQNV5a2<8`Z2815}Y4{z!g;{kT?gFf26qwC|TNJ%=_x? zA=cIpt~8;W`cB&o3vqLUPd-C#LkfX528tOmF=rPY&AvZB?oFi+iNT>ed?Dx({>2m` zI_9fa-K@(zfI_o(aylcrE(c4Pxg7i)D<=SIn)&M0DCBQLT)> z+iZ_WOS`bZ_|0R!GB7xOS+oKsG*`X9%f`4Ktt@iu^68;*$hA`YDNzf09A0~yX1P7Q z|>dC{EL#;;?tXjxfVL&MaKvKRN4f;LhTfxpzbckdhb zk&ea(xW}s2x~+!B_a>FSeP^Tzd{u7WcAD+xmMz?cSf~$J!MTBI(xAPGvxS&^?D%xPILN{(btB|UPJw8xY;6ZhZ0Q&w z56~Xj7U#9;W;A{NAW_gIII_U3iT0#wXn?#D0OZe&(Y4zwl?g1 zpnHm&g@uKk-CI7}HcwctVssy~M5G+t8@lgp3RRy9Wv)0iO-+h(=e|xEt@njQM2x#D z++K$wLLdoEuzU#}1`qWRxVW92`?7_3dGmVvTbd_*@rkKcRy*}V$c)3y1w#SSVp(;_ znHqc&5_)(C?q86I)XwRz@+>D19|L;){4=yDb<5E%B2vbu0SooRT(8C^95KS ze0=<+kclMSJY{)#cW`rj_TRsMcO3h6lBuBv*kNue;`wPZh-UHn>n{>~sDFC(mge^v zLK>V$cuwn&Py~Q8;A2RSAOA&>Z_*G0N$+BhR|KnW#hu%?FKQpuxKJ;G<_nF!kmSxk z+FO3HA4+AL2por7MblGony4t4SG(GKUsPtt6xb#3WD8vzB?DgDwdZXYx(Y_6s`T&N zy<6_R?{)pui;2aAM}?UaRlzEH^D2PzO+i zcvP`PH zCi7F~D_7)lG&0iDwH?oT?CoqGBcYD`z|`O*&ulp)`~pWAi#^uoU?wMY0#vlMvB0%>?{5z% zZwEp>Vecc|o=@eC5xH{b*(13hAkjX@X?lXWQUNN4<1f5nt=2L;2aNy?vNdwzVagdF zl72w0Klq~H?Ew>3?7nJgj~*?Svay*RYfRXO)igFWb$;gzT(V+3*A-S)G~iw)FK)uA zlP7V6D5~v#NwgdF*HWE74{*8#yqgI$_m-J?azzdJj()Fg8_11Y>fcG-zb~|)dt6Wo zSIipn16Z{-V*VeOuU-WrDf{*77Rw^29RZ892zIcr0PJK?XsDE?S^R0|b3lFvlG26e zyuSnWO0Y`H!vi>K8X6km!{ZnfdF_X!FizTS0?COC*K{kd$*r0S6lKbK8w`bRJ zJuYgH`9Hm?=>;rv7nSu#sGm;a=s#{j5IaXl7z6}K5)=%!;G@%|!z486`2+>WM@EvX z;?SJ+4Gn-A6We+WX+2^%;ATZO25~WG3Q9m&nB%@E0w|3^HBC~J-otxmn9gFAf{q5K z=DDB?D-X|Eoq&5ZBKKYUE8HfS_XdE(f~s<8u0-x10_%XV$43;!#KeH%efN$NOSWlY zV?#_xXt*K!ZndJK0*Wb6b!ZOws_fyh{p`BY@4^mki=RF_I3uu43|TV~1Ph|>E0?js z`z^h__cS#FJp+bnU$RM)7QKq#tuYSx>y>sUr6&2xq$_#$O00to81R}oH zNq{pH6GKMC(G1Nu^xqU07egsw9V!>5u-4@FfMIN&UR8*>c5-~!ipK~_%7t6FAh&Y& z)hV#7=G$Luq5R&IBn+Z{1E1eW$Tt~jf31cY7CozjTT|{ZS`XufDn?6yY@&2poS;iC zm&<;+v6&eu71yIVi^*Hyi|*Zf&OQJ3!jCj7Bnn7p_u*RNL8ddW@?O8b0>!a|;(~0` z6L}NhSY2KDuw;{#R#qXQp)Uo!2PaNPGIH_oRKa>f)-WOQa=JrP@9F0QD*Nu;9tE~a zb&vcHzWV9t;i0d)vOe`+?$ zSF96Z90l$sJUqPBjlWz9>ZjMipdo!Daue9x+=RsAomrDK1p^Dz;gdMcZhCk?(Tjlq zDu0*w`L%L16n-h(C4US@WF)FB+6bHi7jDv%gaimgH1+f%Sp9b%;`@v|7x9cY70UqK z2RnQC5FbNd85E!^fc@Mh@cY<4-30}d1l%^4AQcV4c7#%UBN_A@2Y|V4to0cEiF}h z84cgRgG_!{7EMSzI=)s^@b%Wcd%f@dj?In>e}r1UR*~h?91#>1C8gWQ*OP$e7e_+9Rq)dhYhjfh@283#oQ*(G3bW^QhHL_|qt^w$I)8|}CH;z~-yShB|xt6LywORo4& zpQM13U=7SafZ*r+@4!*$5KK}g9*K&cui9q4cI{h?^BdT!aY;F8XK#DHXTgHb5k^Eo zPF`R;AcCR)X3iB)q3inVsirPeD5X0kt}XqpKhO3xDX3dff3YXtN-^)hAy}tWXbxRV z_{*uNg~XgZXXC4qy5c=SXRawpz`Ap)SDh!akW9iqCgp(^176WN3Tc8V9fH_|bs0^p zGnHsw=hQdT?-WU&xB0&n6ANz&wTLUL8{qi6*z#!!$Z0^;H@M= z@(3&obW9hvfPhxsXe#IdjuBQTt78hDSa0r!Bqb$5h5-`=$T}Mz)%o zTJcWUO@!phDW;pif&;fb4lMMIrdCjB;M6mqvS3*Rxw7iYwXkuhG=TM1fmdZ)$|`lQ z6yfLJgt!RqofRh9C&l6OA)ygy@9flkt^@oiDjDr}-i?0BA-*w@vy5$9r7eY^&zv$d6jkIxm#hA?%9 zvfkF#0%cxpAi?9}+(61ujl|*mupH~OIP_T|mX(-<1ga2_na#H8h}_?R>;`g5tD<<9 zyR`a8$X)_AKxwDEyc`B6{|aJyNL~s{O6DgOU?)gHMa9nCUD#bj>}l!kH#IY}vuqh7 z|Ee;Ha&>jO6N<6b+eFo^fn+`y$V|bXR(ocrr5VAtBpaI?-MqY|od6MFdIt&%^%+;eW3Q bTQqplMc=N7RlGkV^nL#eczGD@ literal 0 HcmV?d00001 diff --git a/images/3.png b/images/3.png new file mode 100644 index 0000000000000000000000000000000000000000..9e93a286cdc15ae0ac2a468e5ed80392ef8dc6e4 GIT binary patch literal 16936 zcmeIZbyQVv*EYH~AsvDs-6b7T(kUs32qGd~N(q~;O^Sj7f<>c%D3T%wh@gTXDIHP* z(kUPf-`x7VzxR2*@%{0QbIv$_y=x5Ewc=j)J>#0!HP;=(3+Ks*8Hf=CA=B1UH$o5` zc#DG&VqXFo)C%E+$V1D_3qi;muz#qB0_2Pcg0JRmd_u%D|3 z?2RDuN`4;Jwm0q3Y&Q0e&Tb04t2K4JY|eHHyrz=+qWT_c_D;@P0iO1k0xlTa2Hdoj zwc}M%B$oF(2Pbf~M_aS`xw^P{o%2)R{d3-P@E-fM2rum9Y3FdxNL}+^yTB&}UMDo# zr=y6NtgNhvsJMu@xDf0ina&i81B@b&)FMDHu4|@e(BYQ7*Z%f}?d=rBMa89rM5TqqrQ}8ax1X@r`0oSMtU_`D34Uwj%Z}_OAABXsiW^ z{mX*vY|o*c(JuD?Z9gh5=>N1IaEx;<)^3gpynaG<_72wGE@)mwQ8pnqiT}f`{Of3W zk$*e!|BD;(_%~m}dcz-=$NB;M0Cyqrue*Rx=TyAWPVSzHX4Y;u?OpuYR9t+lJ?#VJ zv3vg4Z{UW1=KKHAoI;%VXHx%TD1O!+|K^gg+qplkk?#^7wo4$ufF)>vk;8MwAEFN{nD03EVO!pS)10Ee~Sq?Dq`|_^o*YAh5up~@1f?D zXu!SnEN4i?C?@9}x!`k-)+DP3Mi;fiE?S?Ycd(<=pvzV@vYQnDcD8jm{958qZ?+-t zkHNl{^CRvX`ytHBPWy3nhhwR(HzW}xvHTSQg7mT>2-_(bN;VL3A*Ris&NTs$HU#|@~I;dKhMcvbUPDDiX^j*rylPBTu(7OB7k0tr| zw4Qgp58m(pWnfa^?cwpw1&)~+8@ry`p11e1K07ElI5;S1=4g6l<>+VY>C|?IH$yz` zZTB;YyHJD_6rF8t*FFU8Hi(^MW_B$O%#c^S{vpVJdu6nuLMm`n;^^_j#Kh~@ufG=1 z)6>r{FS|Yt%+T6z=yP#(tqnc?V6j4be|B}Xcwq6g@HLrr4)@85;^Nh5v(?qrcj3_E9Iy*bh|JigO3EpapxE~ic zQsp)Kd-=P*zW(Z1tyf!w&cx534<9@bza{xt$b9Ar!P~2aLPA3DGiLtayMV2QuWuCPd%`rhA5N9S(s^%;ZqH;$bQjEn}Eimp?A@BC>+Y7RF zarf?7)P;=Ni7N!{7;e!%&hEQ$>Cz>#6KXy_ztd#Bj5~Mj;Y=@Iezg;ql$I_S|1zNKSOsCSTc2!{^IaMF5FF^_l)oTU*E2NaEdEs3qIz>{(X`Jz z%@n@=Xf;@Bd*0422mAZzs0D1MrrWz4j(z41Z!Q;@{G6F_8!F=#6x7>#q@;#8q+Kso z@>#la{d$JUqIjp9l~ujd+5Uln`1tsOvB_)U;oLfq}_rt8CSUtCDv zJ36dc5PH+novXX?>zAjOSNg_)-0=z26P1nGP7&Dn`Hl7if1J7L?mk`{xINVrf0^PI z@z~f{Zf>sT^IK!@{a;n;HO8J6`MosM)7||nAf`1oCZ@R+yP22TL@fyQXdn)eH8uN1 z7B&3*{5ZI{m))Nh?d=6T?;0E*FAj(izgj4zq;%Yz$a&#tu_knCrahILn_EFa;nnST zIo^x?4<9~s9eLLkHP(L2f-0%Y&(DvJj(+z}rcZ6@3rxhH?cV+Suf`^w*xA`li>{f~ z_!KU%`)n+KuUuGIP`gcTR%p^^el5bJpuMGqUqC=tPtUlvv4aqk{&RXdO)2zxUsIqX zq;z*pO-*0l$_4iONr{QKV2SWaX!BWYLL(v~z|5AHmtWnMk))uAKR7t(?(Tl_gbo%W z@(uok(klYJ^!WJBD3a5{mLKE=3_|Z|Qq$1%jg5V9?&0ZEWBLel0R|x@)zQ*p`MfJK zGEzQZ)9HJKE1d9*r6qi|v$wS<;n>mE-!o2BwIGLi_wF6sWPUzhpPIwS&dyHP^Yh!Q z<4UngY^P{wyr;g9-41?eS~xsB{OsAY(o(TLwKOdim5=LFO}IEX#uQI)b^Io;=f*AZ zT^(y_(V0E&zRe8gI6FNZw703fMY%6QN6M4ryfRV+1Gr#rZVuKvF_E>vzR=jvkS6Er z)}A5>zYwCH6k)xF^hBS)skw!Pt92oVw<%bSDN0{_zD@D?@#BK)*~+1zp+-gzraJ@t z0|QNsjgym;W%^7v9T`wh9;KwDeEOu?Cy@Nqo@}tv-O}DZcY&s5b8Cy6m$%hL`RdiH z78dE(i%k_8y3&M@qRq_uPk*M4k@5TR@ZO8i(tcT*)2DS64<#AepJ~Lh$ooxrhPGLy zjEs&NWXKP7cXO<5`lAw!d}S&VRa8_oJ#F{+$k5-~+FD=#-U7{q?BZgP zGiRKgoV2zm&l8f7)%q-71t(o#Z&@R^;wNX3Z=P2=BPI2FtTqs$_*TLhaXQkpqa>a& z0z3kO0v7QpBO{{>#bEwjxL7D)kmKW}bpLC{rrp~H(l!y83N}IxLP8vVzSxr*DtPJ< z&+_TS!t|)78eE?#iru~A(A(9(ed`{Nn94ofQ;+$=bq%FxXz=jyRD6bU!+uy zJ4C6`8eaovx8IjEZroVN(N0SC<5+qmWbU&xsM6acLYQ+I$vI|8NloSDB_WgE<^_V>*DO} z>||tQC>5oXT@PEew6*WtyLXT3+Ez(rW&E8x)^q0Mn<`iFmFGlq!ooflm=+Iavd~v~ z{%nD%$u3N|`{Gc}Ob55-LDIIQ7m(v7)85xn6l6rLSh&22iUP;Jd zz;9W?-JR(q10MYpJlDq1la!R-LU|62T5DE6w7#yyw6(RZjMo(x6p*+Qc6n6UMO`bq zs=T+iS5;L7(e7Kc=tJ9Z_*5H{ITu_sV}nP8Sj^SjN7dub;{-{r`y}xA;1_GUdo?csf zd*REMiGeCU6j_0!q@+4JIR!Yg1#0HELq2>y_VAd;N>vvHokpC;v+5*th6 zOzpO~u%LOLiD(B?0@Gk^&F;+m@#f8&jblN-rnNORG`zjNZ`~?|C_9)6|1Q0_03X>s zt~Wv+ij4)EnmeJd-`>#BAZGhTnURBHYikQ85RQYb*s@MoSUBWxZwt&~VQ~?z1X}?4 zrdl*9{W&yWW+&@8v{O^)+}zw1GqZd25)LEpZgn4VT-^SlE%x)fdmXo9NvkZ-F6BC0GVI*%6`1tsAW++5dOdhj6V}Bd8>q+%2R8=+n`Ey%k zWlT^PD?9q=z@L-z`8`HyDM`tTH_a`@L`AQB2$GE{ud7oc>QYir2nII?6#7deH1gZl z;y_az{l<`EW?0>am)Y5+H~RRmFmj1XN^;BYjaPfOc6Pe2O?)ILCufv)BPAi>WZ&bA zjJbOh6G@C|{psN77||I=MnYo#-cRhZV^Mlq8l*=Fzcnizlcf0f@883OP1R8+q@}ss zycug|91e*vz~5i;>{-LIl)^$+J3Fo=If(2ufFS_YVx-VKXV0Pm9ep#=dXSXl^=ta; zc*uTX=KIpuGHW!L7cY4IHtRfoe1hnE=KZaWo%N~to)@N(HeL};nrKR;m&R6*7@sgRD znVXwiiECtEfhq1Q->g@!bULaGvJL5&m|SdZetjfhRoBo^N&PI3F;{-|>Q!G~A0{cY zr>BS4Amf^)r9saglt2K!&I*kQa3nSl4!+CJCyZyh{rQZ50POXdbKoH$0RjSoFlFVQ zJ-8ASKA6+~OUuh2Q=3&(RBC)zqu;9Nf!EJwhGwLtk-8E_pgttcEH3WDrIlX}^!Dnd zN)?usJy5)}xw(nLJplme8*djb6Zv^IN_oFEBt9)Rwlzia=9-gtF2L``#@wtdjN%=D zYAGoU_1o>KDJig?xViT!#YIKm1?`r-dc~hbMwp$JmiDvtvHRHjs`u}i`l<48M{vTI zk6&M_qWa9e@A|!hn~UpPA;&!o264GsGB#4~oHRlZVmAvx0^BplT!W$Ykx&FPm&ZhX zm`cP66oa9;dCjdU&6_tDm5vXnzmevm%J6euzBIG2Als5$U;FXdxhL;OSJ~UQ!AA$O zz4(;p&o?zRs8?S$Ha5n?LG|7mgRl!QuClVy_M5o4cuPymRM93x49|6eDRSw)*iY%CM`hww8JJ>0Pi8fatbKNr=J8iTQaVzo)jlzkd2X zQ&P<@DOrQi)A+oPCXL&|5iS^VxMfP|ECwJk{j4(;*VjAJeI6Xf(Rl9Y&iO$7N)UEe*YKemLPF0|jHb&L(nHr3S^H;kH#@Z|@^P zvv5n_sW10-G)}4lNMH>;Sg-b87GvyvHxUyP)7GYUP`&?qIbx5UoRl=j z`w(*7uic=l+&PI49(ZoASR7Q}CT9+3*iosJu9=*i#PO^%;G){y-_Mlyr>a9Mp#)KF zX|mKhdoW_QYDhVz#n@nV;u&ezl5- zSXAS2j2a-ih~Z%-fZOkt?h#s?kry%*$(|KtXR9nG^76S?R8-)ch=FvXqFqx}W!)n# zDTzP7dbl$czrVfORQ$adqm2Y9j(2pBtM36e8oVEi{t21UF!eSI3(MI;QokOdvu96v z*ouNX$;mOQrzpDn*pF6w?~IoclaX1}_y{?4g%IlDJdyR{>CN6zI@-ZiH`NOa+=uEz zIcI)f3vROv;;(+}HtB2Wld6uL8Teg|CY}ITg7@ZM_?4fFiI3mj*!Zl}T)~Qygk+sN z0lA?>+Kf_lwj&+4%iGiQ+58S$vNWVMH8nMr968LdJji?>J}92ZrV%j7FDS63T*#`Z z=x=Jeo%V5{h3pDG@+=TalC{pe5EM;JO{WHOi!b8-Y=<;oUQrRQ<(rV6e$VvzfQ}e|sTR^_%lO>#=vk_ADMEEu`lHB$Igb!>c>HI;oN9 zPb3==tFErDh7rPhLI{!7r${2KS$+Nb6GWH@DQ#z~6}V4BTFUvhmZ)=I(eZQReIZ#oyX<5z4d(7$EDCnM`d$Dq%~Cxpr3WvP@7Fav3^q#4F)Gxu z-tOe=Otke3*O;hJ$Z!m(lpJUKw!=&q`B^C`7}q@Fi-TF9?V`5DHeUxf`Ma6H( zA8g&HE?C>tp3&qxrRp6vH+aJE;>C|uC`tx~tH#EW`uB4s4s&t*)+g~}+$faa=J2^N zy%}3leoBr54mY`UtbEVWQRN9W^0<{*DMbE4BRQeGCFy09A)R;@KhmGd{4Eg?C4D+K z0bv`^&EWJEuKE-NKylV@QX*1%L!OI$XAMnFnDxyoQTHodX?IhyRb zhl-@6 zR!+{AamCo&e1B<3LjJ1YXt*Rw*#gB`VvC`qBE)ka|A3zeu;y~m$^O)IXLt9`uU}D# zc@3$|6HCXDw{P3_*vvl)xrm}1JU*H9_%XpVUF~M8+A}I4A&N1=UexuLp@&iWvAO0Y zsVOO{22l#WG;#|_gaBqdQt6>O1?5-IEIwto2%-8oy} zYx6Bj1($%_1ro8#pUuh132IBIlZAxZJ3Ck1H)2Y75Ggw?M3OqWiw0TLC*pgJ@EBz1 z?bdrZzxP{zR$iVwEa{C#GchwSFD)@KF)b}EP18NLFfn-qq#RHKNl7q}O~`k5?-Jj* z$3+}Og!%jfX1XP`BR{;0Jz^^l&+zG;Pk5L&$7(-zSx=vS3~XV?Q;GBIH#MP{fi1l? z3$i!l`SbBCKub_c8pKhpaU;Nj=D&D8=&nh0%AJ9fP83<7!j4Zfo~8YV2cQ2EgCmaN zS}-{o**(Zmz?`%}sROXj*sPC+8oMRgw?1Nz#B!s6fdov8lr0YOn1WL44H=P`i3F4+ zPGfVk)<-))J*r>rab(&0Qw^gpK8d@DSfM+A1I@C@3xM3;7287EQ;)IWsc@z(`n7aHP_G9HtT_bdeI~ahVRS z!A7a99Zx5wnpj8vQ-A#j8Rh(SfLaj`OunVs0Ly#IQe6|!!`|79(eCbFP-?>oAj8dg zqQ=CQXrZnV`LOdWD{BG>Mhr8#M*X${(PiP2`@8X!IPz9m&!0bUwsfo=qRBQS^wuy_ zaliZVDPp&Jyyo{FlK+dmybl{4a^)#p(()|sKU*F`by4H93^3)}%qCe75i*Mz>Y!v1 zhnrB&p&_MzEnSlf`BeS!13W8N*TNkR@du9{@wBpf-MaO)xtWldIGlmwj2wVU^9mPj z0|Q?#uZW6ViV`A(j&Ea-7(-5%j(C_LWn37zHejW1o743APo6&2(A0#LeKbrIy!nv_ zv7%!TzvAvgqE>q*I*k_0I|A57+Eb?@+gWb-A0KW5GYQ2!v8&c9_QcWAq)?teT3EP2 zY0*fH0fxLH-Xz;_@U!yjNls0mneC1acKzbAi4+uSpFDXIuee7zoN(Rn2oIsLwvWsv z!fZKIiuR@wW|!JF8QI(sJVi^ZBZp(>=(r8Op6`qM(!L&p`uOqV`TI;4FJ26*(BN;$ zLQdP8A~_JeXg>s&U#HB&L$36z1lzO{8aEr} zIM@I1fD%?UTdYj&opg5FlxG!jo?RM|@?=YSO3E-W?9+6kZGym-cVsG2xJGe3p+cjQ2`s8JP@ zb@j;zBJ!)0U5&vrzK21^gYE63X7WXSXFhQz{qyGuW?cq4gtEvOrND=+l1?}RFr56_ z9f>)ELSXfr*~qHih+Ca=tUW`fkrrOHJ61{rE?Z{XM4r~o1tg1$tu3x=}@(js)Rl3r`A7x`c(e*Z8n)ozLGHlrJG@Q8REo3EdaTu z+sY81?E->Ap`hHC@&5X8a1baex?$qTJvIDqV`JppLL`_OF)>uhSn`QWR2VHTei4y@ zt}a<@V!Vj+R4Mcbc+0H9IRYAlQ9Os;7aPLKcmk+@xT0>!Fl8G;{jL2Kk<;|roV&Po zo_mNGp;O+NzCZO^I`^ZxJI9g`8?6|G!ig8E59@H|dRo19tS6;8VUm!Pres=5T}Q>k z%^s)-5r%rWQBg5F>_Dh{WN~WqJ3Zl?%HC2a!k$}=!ukNcJsh1x@n&%l>BrewWBkaytB*Rim_fx2%!scxD zym9mpypD+#(ky>ZOLP0b6H%g|D@#U3h7*=mUXF$us`}5SM=vSnyHr*Eh5G^QTw5;8-NWF0Q*3F-b$y~v<<;9&RXMlJ!>d{uXD3XnX+ zS>O@_e=jA9?fC-63jtUSR0q_FfropNZ)y2ILE!;eOe~iFm`8NtLzCLvkx5fDz z0ds`1cszJ}6p;JHw=+-%Svx!LZVgxt3=CX)n-?Ag+2;w(LnRdDQG0Vk9tx}h)-SRW z1IkA2`}e@`p0}P`&8n*M2Z9C$V=?}c;N$o2jBn%yX(>hZTp0e?NUgRfPR!lA3=nz% zN@x`5K;{OhK(w5NEv{>7ri_h`h%sx#r`HDJ?_d#tGoKm$xBmX$eWsRGH8sBPS+xbA zcDj_O_Z7f#R-XjS^_Md*4rq`c1YxLlTO0g(xFyfPr23onpyt4of_fy~5jzeaIyyRD zzI-`&s;050hx$d@FH{ z15mTIwRQ2tnnc|hXQgo}hz3lpmfq}r9e@JutuFE1}!TTT`sB0>s!!x@lt_`6$*i;D8T(G6K+N3Ni%h0k`1__WZ~ zQ_Rf3uc{K_BXSxU>^taytISwR? zZzcyKK*m7wFkFz5k{T42FN1`YpOBp;hmGUO()c)GBcZ2Xon6&~&%6hD}&qso-g+u5z$04ip1r6b><1OIH`b z5h{5vl8AbZ@JuMaD0Uj#de_-@coX z!IF$NGkeMrsvsf~&#$|O_qdh2g3u~o8YO^%GG0x)m7JNPt2w-k8-b+(vTVt53d@Y< z05f`OcM*yKLxM^&Qc`7fU{v-P-fKk5FCy1;5Koy5t_;5pPM^cwNga|HtO($uyrX{i zcm>2S!=;PP|4DukDUsVB6&s9 z5QB$odEIx^V(rJvvih##7DGmFI3AaLp!bGI8Hdk(W_p_M9ZrVh>6%a2J@=xhcwodT zEFqtGZhFg|e@y)Wg^j_c z4lrV|mdUV`)^2X(&%R4=YQ|v#;_h@2Vf5U8`N_k5(&TX=3Br^ax=~lC5#wB0hc-IO z(Tvja^2i5yTqv;Sur#*bt}c3cKhcoU43wU3n=49;8--ch9uKuO{Fs7(t`GJ2E&jWJ zw%AMDk%x#n$H~ieYa-X*XW6BHzl|o1C*H(;^H`oc_lc0zza5?bnbrt^3|)m%O_2V=qylr zPsQ=;oGelE$87elDBvq<{Wng$;Ztn}7OBpO0xDd3os>_+S#RG;3JKkN)o0iA=@Yib z+wsm7gUkgY38k~9DFOk4b4~d%o*|L4tN;@EEs57)EtZ_&Y;#Ksab3|_tlsnt#9+_` zw0?vea;F3R9OG80?0<>x6b~=&rLt4KKKCI<#1t(l=km#~cHweiZOQl~LQ_XYP5sh> zlHa3bOr5Y6*wwdhA3n&v{@yQ)cT!6^#x^VG2p-&F6TcklHCni#%k~fC~-N zy2Mg!=FSnB{5ciM`BOzaoYTBW!jdr6oi)7{TrGrxMZR9-E@cYPUBIqDy_wArduj!V z)z;F=S};+`Tw7bK7a0}`2naBIVV1welZsU1=hT4`owF0mT!thdK%s&}vAzg8pNXX< zYn_`Z5;5@ne0;t|NmUg+*K^Q>)Y9K4*F%cbNxAAC&V>PWDQUyLpXT!AJ_vSG1C2e$ zDr`guJ9>3Yfzm=+TAI>=D2qxN=pp9Y3M*r^y1OZ5%32d1zPb^7}6-_a>J%=GlyO=aur>!3@aUyn$@IUv&Q zc*{*qPv-!q#-V>(T7?Qo6aa_@>x(U)pgTnw4Mw&`_>l_(-4{0sOzG(rgLdNRCDs;w zc-YtwoxP59Ikcd_DPm_GA%*8fMJrQN5tfw_(4l15t&@aMsuq|O;>?$C#=%do&E6>I zWGV*VJx{pA%KXZ`1U+v=X(1M*$aD=(^_bxk z-v$OUrQKP;lZi-4@w9ad3kwHN6h>ep>5tjj7tfyIaNMK(wY_~igQ$=d^oT6aZCh0J9jO<=YCmFe|#r zYVnw@9uGqa6f|wj3M5(;GypjgnWiTz4+PGhP43^yU2i;XiRqmOjD^{jozq32^#r;M zvYgj5b~{6la3wh^1u!*o$(=XnQO$TBD_npFoXpA@d3=lk$?1D%;L$ zvMyZrnE~z!igdPam=2lquhn=k{r&xrz$xbeI8Vt_wEXaZ=7<>m3pj~dk?bh8NYHj8 zRB{D?;eHa2i)FMir*-Cj&j3OzO;=lO=~Ox<4Xpj#+Ikb32n-hniY=QY@U=+ShQ0$f zRMiIc$F?TnEi_-$uQx)V_RBa>3_X%R@z6~+V3V%`S9Y~_o27TzGSU!esT()=NBIE+ zp>?%BH#hTfap5hfCsbGO!3+^|2+e?UEb|jXtqh8hzdz`KFJGR?kSfZ~{^2_wd|O3} z!B6q{q$)7p$p(xMp&u?C^h+yB264OQc%`El?C*OL(E5Z^RUqO3s*qzx8tP@~huxXf zM9%Y*12jMusP?!*k;RK7YN`9Gy&(WjXD(y9V`?2(g%el0D=I53KLpXIGW8y@t2#B_ zr9p51th%<6tNR$JRj14I^z!zPl-aVJJ(J^h_V-!kd`RESL4Q@<&Lm0Ov-CSLG4(+* zEJu5iR?R?-R)rj%D-=0`wj1)_>z}|hKLzP=<{{W&m+ewTwIvx>FLNwJU(w%*JV(VyGy4! ze|UuK{(P(vOMCHyXJ&o>@yScquakSTBkVavMW8zr6%^<;P0B&*NQpHrvp_5P1$Ssx zd|OsVm3pnx1!SPk(XaLO^;oeX$Y>h6y~G4Q@j?cv91kJMcXM{8XJnMKX&@vXU;;`M zyz!$WQ-5zSbl2g(nS&`474Z2|Uk@0vV`L=llVoER)54Jz0o2-0wjDsYymP6@9D36{ z4|Z+=Lfzwn7BPeLa~k7&;+`{Y^wlDZ=%^SxkLBUF@88QS;{wwN#E<6XZA{Zt)a~0M z*WM*O=sWcTjCp6)m5Yn3p|R1Lb~z5}9~+xz1qA}!1-Gh3V0t2EJtMyX6?V&KgHp9A zp0#O1+?4{7P^aSl5)2tVQX?fQDg(oLdFe%c-Qh{qoo#ta)f_#`8!j%9%mn$_+3Q{B zS;?4WM1_TmYHKY%Xvi>iz^aN^ejwetukHwNn^xrd$jC^XW%a99#6A~zhl8KatQ$VQp4c1FUkIibEDN|z}JZyHR)VZgVT zS_VTyjs}$2AdLqEoIBc&BV^w>o@Gvc_>hcMk!hGX`SIgQ*Y8rY!hG1W5P*HAs5K6= z$yaPEm}%iv{M?MhL?|DLa>rDlT#z0Q+Sp&M+m<}Jx4&O(QS&H`w-vfliOVuUmdR8K zz0kbTN|+5@Wzfi$1~eGVmZwM==>Mx9_V1@vsS#oYpfs6VICa7*L{?BZm!zW7PZHum zC(9)#r#WZ>42WTdkb-Td@bx{09u_MD0w4RfCt^_27%pHHJ&*!V? zdH3KT-*P;}K>erUjqUBOpjVzaaY9CD=^-#1@*xLw>cii^%LxjSs#g;Lsd*v&ocG@5 zJT!us9leK2`^Ae{NJjGV@}y*B^HRkI*cRIQ`mEcTdV6eqsI^__)iVMD-${+@jG$m! zM@JD*jd0lO*T$DF^+3b}U@7a#HE}qW5w>Twr048>Cd|DSI(!`L?Jt{{fSd)zCqzhQ zXsdN~#l?;A9rGs`u3qs!*b#FP0^BnN5+jr~z(v5?gWd@L;ie$^4fOTFB3&~zHHD}T zU4Evfz5te|lV>c3J_PUA9Ur<=I)_sss<lL$1_?D3sH*)I%GWN5`op+GjfFG~amq zP|X)Rv;oY6>bxN!4Wj#(FJBBf-+k3V05G<;w1fxPYdgop#(MeqL|5RvEGPihjjGSt zo?~}+*TKO7GqBijtuBQ50N)|k4%`Qb^e7|VNjfJ2C_#1zBn4QZ6CzRy#U2T3)?Z{n z;)*}D3n}H}^SVLO`g!FR9cb?XLqd_U2|)P2kS2o)yqh+k2DEEhXgwI4TJ3%w6Ep@` zRHKQ26r*fuYl}>G&QEq3F4taBn@v_WA<>G<&9%RJmBCp=T_POVPYDT$HO7NG}dByn+YK$P0+4pq1DpTk^;zs zh6+prgeVB&9T^IY!<60HIy#=7o;O@wqoi_9gYew`2&Wcg2MIv=syLBnAs<3#>IpIi zJXgYvE6~Yu#J=3kb`BPAC+G;=Q!8~;MzC~}|Pf59IV>1alKsFis<1b${ zfpd$CBXv>>X!`t_Q(is@I61h1JJ3W6H+ZqRR-lQG_WBMX|c_t%Tkh6srBi1;^R+K zT#uHD6#Vr0Ge9pQSB=cCZEY@2PERCn(w4K^twU4I_E=zis$!c{0>Z-g6(;+hmY1KUprA;& zvvwEyf)mrz$rMA^53$@8m=n;-7*CVE_VBvplabCv>yT zdR7_NjqS2b zBB2)S0Un8nkWx3HGee<49G#k&sDE7}5zH+n)&=H2k9BjPq8EB!a>bWlN`qz-XstM+ zJnT9??jj>13X{U4h7&N#`3OclQ?i+4!Qest0<9VVTvk?Abo$k4@2D#stbGPxu-{3| z1br+(>H%bfhWVClu4^71CGqhFfLY7gcR-V`g08oakPjf10%=e#i@s=LQsK~^a=e>) z43z|?BG+BC2EUgcxNHE~0_|15U?w1G{({B(7F{nIusH|1k;s)2WSH`XALPxJhXS@X zHd$n-h^DE+t8e+?nG#de)5EmTax^jKAJYv zXBE2V#wMsfr@a*Gb8+~YiBPoY16ur(Lc?<$E|;A(x{EFwT_j0&>EYp+tKb*>#=$4R z5qMFhtR_J~aoeB8Z7k6B%Wkk@tcvL@!R@E3vR)m>dcX0kIspGc;egc(R#c8YJQlIc z05*O2?ag@>u0!Zf-rw7U0MMh*JT47fIm|n#Sa!o|R^3>A68bQj?lXfQUf32q4u^6N zP$Jj!kP5HTG9d(>$`vqqt;Lgsos|51t!=^B4FGbm48Y$RTuFPJnpzWjd<3~3Dr0nK znGAHI0jItU(bdP2MDQ(SX_!@CA0OeKpEz&cyn(7}Ze^t-oO)A`v_1`FI3^hn&_Zn? z*Uo4BEWQ^Pw+W@*_O|2C;IQM7SC_JPW$>*Bc;F0BsHT^t^c;zjwN27-_o+IB0He5W+vKT#rI)X7KURA=H}i z5YD^6ZMSMiGZvT}IXSuQ#2||ETu9IWy7$_Fn@T>3UsAu%Dldz`$D# zqWUz@N%T{txT)@_bCg2#v<$i+Dj~4}E^f^KP$!o~F=&o$owcRCee~TeTaHBRgDudG zAfm|oBt1PCgev}r3=9k@fz?~ZBLTDpuV0VU_`1XOb#-;CGq!#~!z+*wumSh#xV?YY z1fd1f9t6o97>i1mAtOu6Iy5UQJln$z#S8;OweM=iVwf%$#W}?!8n~;GI~&lyI=ekF!s-RR$szjN%?Gc+ z=s_lioOAc?-75a+pFj0MLebY33OatP%l!o3jbe!0u8>+rEUaM!*Ngw{4N{xGRBYd0c<+u?1um$KK~Ov^36PoC%= zaF8S);TH(f2x;p^1QOkRNfi7+lBc&nL1O$~tpwiNogm?`&0Ntua0kiDdlxmBWJ|TM zbEEpYX}U}3>ab}?Xn_Jh5`~D5@FV+&XhjetR`hDYJ!4ow0)&FyJ+y2Ljem>46G6g@ zLJ8DTPzVnXmk(Ew4+!>DP}0=YR8Uk_P*#=$3AvC+e+n@|&Oc-`Xkf#y=rAOOxCMI$ zQoIBF@eG~Bg8`uwf`kMc{#NnVj*79Owt zOI}OgJ%sW2>m|0Npp}PTPh{_vHw6-dLr8X!fh2;24JjlbG}w)_qJc5#w~`&fBqD|6 zuA{7|tR|*`=8t_Vmi*H= zqX>$LyN-sEI`N=}$2K`d6%|D}RYi?$avDU$%7b5rby^0;^pwF|HD##&(>D>o8teAg#`XB*BIHblJbl^ zfG=PbD!*3&Pg?q+6t93_oqa@qUlKVIuTKsm29v1T49Wlg4$^SN-~UHG1)TWf)PI-~ zK@9v`N`jcyN-6~e+XV!Wb#@T_!-#lE8vDw;#?#7YIBwuGLYs6=4E60IE`I54m;S*n z)Htc~XZ8o7y)-4F-F(Z5mLPyslniUpt znf~X?|G%A3$1fltFgs~d?c}d^q1B_}mCv>~pIeSb?MLId(5ymzef=w+P%@e9cK1)tX2U%@kV8F%2WgwOs8yg!P9ZjXq#{{tHu%bm$U|?WO zObm%c3J(7E;k84_tL*G-gD3ZEYWiSKUteEuud3i#1GSDxW||X$K#-A-cG1UEP7cF&RDioLDWFV!Q4)Z;I%n z_wL=>ym_-x`=N0IG~~W}_wFfbOh7;Y*gYgSF~*8Ec~Pm(eKg^`hSN^xV61XQE7FBA%HxURqT(Ge1xD^eh|_ z?eJ!%1^D^+Tz&4<($)qRY9$V!*$20)szii^o#JNAmzWV+UjEYOZN)?H!A*GVrL0Oc zvSL%U(OSjxoLSYn?f~nFjjO6oZQb?K>x`hBTUxNp^UIR-6l>d)mrm|Jyz!vw6-j@| zT)Z@!>1qvHuHvOL^#zjOV<%i5c$`-x5N3w`+s1e32n>A}-x71DD(bE^!W*ZhrIk38 zS(uqMoXtj`j>zG;HaR%ZcN|}fV`jGE#NtCF5m$P6f24$x((~5Vvs5lLSI@_FVwabf zm#HaR+Y%z@B0bq-92=4{GQ8Z}b`V(gb|1nbYgRTkE$R!*=`>XuT zL7@3C79AXau<#Xz_DJ(i;gQ^0HY7PWH1(4woW%bHWn=j_?D zFE`W3pz2tDzT3frmyR4^n!+C}w0e7-rv(>RMKg|$*6x4LuE&l;%zSc=JnPpR5O8|C zJ@pBwQsd{sjBfZ>p(&nEHVzIB2M%1Pk4{l*ED9^Cs=_HiGJ}IDl)>=2@^WQPPEO)+ z<_JtX7F*DL+i#xE$I*Qn78Oi??UTu)YZVk0=4gg|TAceHA0IDxwxG1M zEhT@?%GFmL@;THc*S-W2^qer^rB5iml%WXnYR#%6QhCU{j*sg`#lgp?w(L%`vYZro z(b;L4qZ2dzr7I;RrMug)o0wusf;klx72b`g+5m$#Lk6cSxD;f+zcoG9Gc#INyH!Hs z!mO_j?Wij|tH9i3qe$`2S^nzk>Ym$vJvZGN>yL95dtzMI;E=4Lkz*0LHBMYWQaR= z7=7=)8!9u_?KlWEslP$v7S zJ;yR>BQI}!g|41T9@2{-L``$BvbyHniJVu2QWcbuDM(LG*tDHbSh)Yqo2SY=dLuWm=um6c_VmfFS^${dJ3+8$0d^V0 zm1I*atDsO-Rpq%sE9}dgqeq$3PNHu#Uc@S;teiC2wcK!$8)PSrV3FwAjp_*`r_j); zm&LaGwql>IW<$(w0RbX{f=$XnlkXp2xpGC88#Bm^Lx(qL24B8)Yg@&gYuB!g*Dh!8 z6QV6b7At;eJm()IC zUiSR(fEqvwk*JI360ej?n_2Cf~mFA{r&mMM(=t)Wf`lF2FEuTFLr!q4$M@B{< z(tD}`E(8dm5V!deoy45nT)fA=XV0D;&iM)X_;YcwpE@cR7Z<0iTdUq5QU7LhN?Mu^ z6hGtSoSdA${`xC3v#ER?#tdLEc^73_LR?&2n{nNT5Zo%<1XZ}cezT(D#K)Y$j~}lT z78YK)0=QS6n_JiLfhmI%0!`f8+Z$q;l)TNgj+Tk)Iwk6lp|@{cQ%$$L zbv1w5)YM%Y7Y9X}NEE$bNc*${gW{x9uND*>+jp^|;(Pwia}m+`>FM;(IdM86pUfWa z^m!oTQ2M2-!m~L|PImbmMio+hacZnbT~6+-xxJB*(ZclD*?GQr10|oAMF2?UiL zg>|Cu1buk2W}}+jo;}v>SXz(XL+1Ye{+4op%`@{JCf^OyD@#*52I&8DWN z_GQj~E3^rkhe_YNBQevT=@*H87F<% zpT2mZEHCdr@F3>v*RN#b^+(If#;@#7R}1L>^7gGBfxS}W4l9SY6wj$+$5wgNdR0Mc zp5)vMrxshp#W{pD5?d8NS=rbqN6$_GHbLE!W3@+h$?56*6%}#d7;ewFTaH<&sau9C zbQ^;_(ubA%Iwf8}^ z<`lleCKu5p()f|IwDi(c`;o(kEwZ({3d}cpBxhWRkADnK0~Gck9va)HxSa=0G?b&J z=H}(O;{@uD+3)ST*RQ$HYL@^ZxxT;Z5>s8T`)oZExuX>I_Vrn2YkLO-w3Amoxs;!; znXoYX;ao8bMzfv`MTp5F7MA0HuU&CTYFb*<@1I&<+R`pU8Rhi!^h~RbsdYqZYHB+Z zgoyiek9xFb-U@tYFe0UQ+X~gOd>DA(B1dB&STy^Z>NjuR5bKXwTVlQ)#UdR6{<`MV zbvima-#@o|ZZNmVv-&bnJMe>_juyGu(5x4|bW`&kei^iTcXQVE@Y{8Tg_;@~LYl!1 zZEXouuEdS7!iT{)S49D5o;s+91F;K!9@UrLkB!xYjX8n={tN?}u|?OesS0oa(Uh^j z=E^5+pyJUoL@yJW;=#*6lpOiUmw5X0>4&a&LOv(h6ua|DTR`Flyt>K@0wN%=V|wQ8 zY1l=2(w5ytg=kI>gJ4+NC3cPkMHWW>VvL*_#~w}xD13cw-wp!qDRZ_OE?PTg{$+P} zCiPTW^T2?vjLc~&TcS49HLtL+FpY&ypLgnphpeoStLm5KM0QTjneX58v$5iwIAmNu z+WZ(*F&!G7&_RlsXV0#gVm(}3T3T9EbSG~4flV%Vb(=>^hDDYZFE6h( zCwBcYEXp^Tso3@L<44#I6H`-ZNy(z3B1tK!xzF@BU0v088Q1*${4oe&7M9atk5x!b zDXB6JHkO$6$L`VUydFM$XkudW(pErJN^1DcO*cR+U{Bf)2ykvCD=Vw4Y(rKMm6VjU zDtQG#a5N8Avw3HNcJ!>`6wh+2tD=xbpleq-2?#-fhiYKRYVL2}zA@|GaV}dPDJjD;F#(HW*45yIT2gVNPD?lfO2qR9 zC$#gN@YLj_iKV5wZ_~%-bS5Sy=qOq)@_;w4NQ&KGWd$_!Uif(H1@#J!nwW(!Jw2LnK5@gmrKqTA zB@>OpPzd=|ApA}N5Ns8PKw@%oz0O_O1#$5~c*mD~ml1kd&h_hZ?-2d~RGZAA+caqz z8J0DIhIe>F06sG^1g=sNK0q|xU`4TvrB9ME!&CElJC5`5@x4fx9IF~gx~A*Q#=#+O z#xl%Ut6@1dV*ZD>m)FoOUwRSCyKF*~8mWor$V0ttdhOYS;!FB3Ku_h9C&nY|fNiD8 z?n}zdG$lMkj+&@1xT3k5G{$vJU)6~_qA;5c; zFhfOVrr^lBJAfe&K_gO2VgP@-x;fG{4rR`E&sWwXILC(9(Q($kpj*~m>CL@IRh|iP zf@&Cdo%{B&lIgyMlatV^koR}P#!QpglNdq!oUaZH_wT!Fq@l+xtQ&jZmoA^=1w=Z@ zDN&UN37idMKQY45V`dj%3fZ=z?Hya4tK#CfW(B+9)}W6~yh%>0Ygz>wm>_*NJ7L5n zBz$i~xY(zES*!8`RjDJ<8Jo-~Z?)Yr!> zSj8KNo=hJYp=c#J!Lq@E=tLgW7zE5iyxQ2COosnr*$$zH?Vu8HUZqc%UP47ygcCfM z8KSPZ4(51+JUi*G&q=j~7;zRDV-VEd#LXwMxq?aV1FOZMTt{n1OAASjM zuC4HFFX!A-@(N1Emyiq3fC4>!93LYOmP1g)eQwW@=n9+t(P`OrT#ftL5+`ssXIO+w zOw7;6r}?`+8ygt`Pn)i8eO_oOlH_D^UzWRJ6%K6`?ZVj<+u0VF5p#Nkh(w}zb(Mdg z*fX5T4h+p?S1D77kh*7^P*QIJbm$9nbLS0R?ov*Q^D|tz6~I!on&a(<51SwL94fHL zGELfVZ*OmHt-pQ^Be0WS(&?V=?umwFj<-&U&x8Vm7O~+~RFc6tKxzu0wBw&`(vFw_ z(ra>Q`PB_#Jv4tNm#k48CmS-m<<^)0Ih!;zHA>&ErIlp3+<9Cib{B69(1WX9jLgU| zEqNtOpr(j-KYqMLO)X7)J^-4=triz|r6%wq2C2t1Jbr9#)SjVqWTe8OtouRCz0_1* z6N<06{+r>dJ4 zRaUBAr6g(rDgOYo?L=53i2e5?2co~P`OpE4mTlj{@}qb4$JeZ40!;Ky*}dWM0a`h_8WesHL;e($CLtVP@QNyRo6+(d*Z*?;F^2BkrQWyzt+O^U=r8pMfUro1pVjzV=4eA6o;Jk7xRZ zwgAU5D8cUtimq(XqKOVu?gE03NCD15E%Wgixf?o~VtNJ7Qdw2yQsL1GmE&>X^k&-vXnz>QStQIv*G@l?NGbFF8y2(Jo^dW3+D=OKnFe#4Gs18_Xpb1*473$ z^78Tr?o!r`EebWZLVfu~0|%kp$%=`ItrgbZMTpn-Ru#PE_;YbCWJF)@Ti`l@7BNv# zOuM97vciK|tY;h-GqhXs`uh4(Qc^E%pPJj-r`6W#!aBHT@d6pUgMxw@6E;C_RK#*V z_36`Hy+2nQJcK^GD0K8Sun=gDuLv7%T`QuaP5t2oJEQ}s?F?W5D85p04Zr=M)|@-X z8T*q3Lu6-d*$pQlw;az$Q|(_ASmq@7O5Xay;XgRN|?gq)n|p!ja>TZW-+mrnP`sNQ@8t2W z(h?G%0TJDvoVj_UPC;Z@R8G#XB||ALSqUqcIA~15C=PZdt(CYQz16wDrQc~V8i>x6dz%EBFaI7<&LIH-HJ!`I~r^kMu z@MG@#XC)=2oak>RJ9aQl$##@i-Vssbs(^NTdAcWHtf#6__-IyAQW6~DK?iFK9n(6U z#MNTK^FM~+o#I=zSaUvl3alqOCMIZ_1&Rh&It!=p{h9u#kWVe=XKr=8fB$}FAWm0I z%%HFn+DbY8QVdN1f)sMt9HJK}o^sTMz2@fTW@asyO_Ke4@90AtM_FcJoRN<8>Y5I^ z3N;dqt}*NHJRE0%m*kKZi@=Sp4hwUB^rXpt`(NB%+}k-7CH8JK%CP2=k--@~eBYRxpX8yKOc@AJ3DRf zU9w@(!K$EBmw<8^o(SV%~yvS+NZu`#wCTk9~ez_LW9Qt!$( zm&?*$zaFtY+|$zo2j^F>!U@dv#p3F_y=oKf9-f}+>gxNizaAO!sr49|qB2eoN!Ged zhGMrdin+3*7o+e7u|U?Gc(R*Gh<8(;mRkNX=P+UhM=IEU1WJ#Sso(@FAooo zrD-u}RpB5HJkaM~)idVo?3|sI6O`Q<7K6#&}`2Wl%v`H^K;aHev|uu^uy!>rc;rPyfVJZVeqpmGBL6+EH!XF{9haf BE(ZVr literal 0 HcmV?d00001 diff --git a/images/benchmarks.png b/images/benchmarks.png new file mode 100644 index 0000000..a56c863 --- /dev/null +++ b/images/benchmarks.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4d9ec8d8b1e581c2e278548a8efdb04336877bec73e4aa5713b815dff96999e +size 1143594 diff --git a/model.gguf b/model.gguf new file mode 100644 index 0000000..9cb0715 --- /dev/null +++ b/model.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c6f2af03a6cc100a28e1f9aad10132da969cc981b1b165d838c1d8d53021621 +size 98351488 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..6e3d4ef --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b493e1620b18a28bf7a2d369a784fba18ee51123f5d65a8ff6bafcee30730d3f +size 182304224 diff --git a/prompt.txt b/prompt.txt new file mode 100644 index 0000000..3cdcb0c --- /dev/null +++ b/prompt.txt @@ -0,0 +1,5 @@ +Top technologies in the world: + +- Javascript: A programming language. +- Google: A search engine. +- Python: A general-purpose programming language. diff --git a/reports/appvoid_palmer-006_arithmark-3_results.json b/reports/appvoid_palmer-006_arithmark-3_results.json new file mode 100644 index 0000000..0187841 --- /dev/null +++ b/reports/appvoid_palmer-006_arithmark-3_results.json @@ -0,0 +1,157 @@ +{ + "model": "appvoid/palmer-006", + "tokenizer": "appvoid/palmer-006", + "params": 91131072, + "dataset": { + "path": "/home/appvoid/arithmark-3.jsonl", + "sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435", + "items": 1000 + }, + "runtime": { + "device": "cuda", + "dtype": "torch.bfloat16", + "batch_size": 32, + "max_context": 1024, + "attention_implementation": "auto", + "compiled": false, + "torch_version": "2.9.1+rocm7.2.0.git7e1940d4" + }, + "primary_metric": "acc_norm", + "results": { + "arithmark-3": { + "acc": 53.2, + "acc_norm": 52.7, + "raw_correct": 532, + "norm_correct": 527, + "total": 1000, + "categories": { + "elementary_school_math_continuation::addition::grades_1_2::easy": { + "acc": 32.03125, + "acc_norm": 31.25, + "raw_correct": 41, + "norm_correct": 40, + "total": 128 + }, + "elementary_school_math_continuation::comparison::grades_2_3::medium": { + "acc": 40.909090909090914, + "acc_norm": 43.18181818181818, + "raw_correct": 18, + "norm_correct": 19, + "total": 44 + }, + "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": { + "acc": 47.91666666666667, + "acc_norm": 45.83333333333333, + "raw_correct": 23, + "norm_correct": 22, + "total": 48 + }, + "elementary_school_math_continuation::data::grades_2_3::easy": { + "acc": 32.55813953488372, + "acc_norm": 30.23255813953488, + "raw_correct": 14, + "norm_correct": 13, + "total": 43 + }, + "elementary_school_math_continuation::division::grades_3_4::medium": { + "acc": 87.03703703703704, + "acc_norm": 87.03703703703704, + "raw_correct": 47, + "norm_correct": 47, + "total": 54 + }, + "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": { + "acc": 28.000000000000004, + "acc_norm": 26.0, + "raw_correct": 14, + "norm_correct": 13, + "total": 50 + }, + "elementary_school_math_continuation::geometry_area::grades_4_5::medium": { + "acc": 94.23076923076923, + "acc_norm": 94.23076923076923, + "raw_correct": 49, + "norm_correct": 49, + "total": 52 + }, + "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": { + "acc": 64.44444444444444, + "acc_norm": 64.44444444444444, + "raw_correct": 29, + "norm_correct": 29, + "total": 45 + }, + "elementary_school_math_continuation::measurement::grades_2_3::easy": { + "acc": 68.42105263157895, + "acc_norm": 67.10526315789474, + "raw_correct": 52, + "norm_correct": 51, + "total": 76 + }, + "elementary_school_math_continuation::money::grades_3_4::medium": { + "acc": 37.5, + "acc_norm": 37.5, + "raw_correct": 24, + "norm_correct": 24, + "total": 64 + }, + "elementary_school_math_continuation::multiplication::grades_3_4::medium": { + "acc": 81.08108108108108, + "acc_norm": 81.08108108108108, + "raw_correct": 60, + "norm_correct": 60, + "total": 74 + }, + "elementary_school_math_continuation::patterns::grades_3_4::medium": { + "acc": 64.15094339622641, + "acc_norm": 64.15094339622641, + "raw_correct": 34, + "norm_correct": 34, + "total": 53 + }, + "elementary_school_math_continuation::subtraction::grades_1_2::easy": { + "acc": 37.60683760683761, + "acc_norm": 37.60683760683761, + "raw_correct": 44, + "norm_correct": 44, + "total": 117 + }, + "elementary_school_math_continuation::time::grades_2_3::easy": { + "acc": 100.0, + "acc_norm": 100.0, + "raw_correct": 55, + "norm_correct": 55, + "total": 55 + }, + "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": { + "acc": 26.08695652173913, + "acc_norm": 26.08695652173913, + "raw_correct": 12, + "norm_correct": 12, + "total": 46 + }, + "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": { + "acc": 42.10526315789473, + "acc_norm": 42.10526315789473, + "raw_correct": 8, + "norm_correct": 8, + "total": 19 + }, + "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": { + "acc": 25.0, + "acc_norm": 21.875, + "raw_correct": 8, + "norm_correct": 7, + "total": 32 + } + }, + "timing": { + "tokenization_seconds": 0.19381939000004422, + "evaluation_seconds": 17.806661402999907, + "examples_per_second": 56.15875864475802 + }, + "primary_metric": "acc_norm", + "primary_acc": 52.7 + } + } +} \ No newline at end of file diff --git a/reports/bananamid_base_1_1_report.json b/reports/bananamid_base_1_1_report.json new file mode 100644 index 0000000..4aa9a20 --- /dev/null +++ b/reports/bananamid_base_1_1_report.json @@ -0,0 +1,19763 @@ +{ + "created_at": "2026-07-26T17:00:32.578538+00:00", + "benchmark": "BananaMind Base Bench 1.1", + "benchmark_version": "1.1", + "dataset_id": "BananaMind/BananaMind-Base-Bench-1.1", + "dataset_revision": "main", + "dataset_file": "/home/appvoid/.cache/huggingface/hub/datasets--BananaMind--BananaMind-Base-Bench-1.1/snapshots/266ef5dc7c812b83c5205235f966ae2f41b3e5b1/data/test.jsonl", + "dataset_sha256": "2f563bb46df778ca494fa20f994a8d3045d4c51fbbffeee433764e2813abea21", + "model": "/home/appvoid/palmer-006", + "model_revision": "main", + "tokenizer": "/home/appvoid/palmer-006", + "device": "cuda", + "dtype": "bfloat16", + "model_context_length": 262144, + "scoring": { + "task": "four-choice base-text continuation", + "choice_score": "mean conditional token log-probability", + "chat_template": false, + "generation": false, + "add_special_tokens": false, + "add_bos": false + }, + "elo": { + "scale": 400.0, + "prior_rating": 1000.0, + "prior_weight": 4.0, + "method": "weighted fixed-item logistic maximum-likelihood rating" + }, + "summary": { + "official_complete_run": true, + "cases": 350, + "passed": 231, + "accuracy": 0.66, + "weighted_points": 425.525, + "possible_weighted_points": 668.1875, + "weighted_accuracy": 0.6368347207931905, + "overall_elo": 1124, + "overall_elo_unrounded": 1123.965054004459, + "categories": { + "language_completion": { + "cases": 50, + "passed": 47, + "accuracy": 0.94, + "weighted_points": 73.75, + "possible_weighted_points": 78.5, + "weighted_accuracy": 0.9394904458598726, + "elo": 1345, + "elo_unrounded": 1345.2360136505486 + }, + "commonsense": { + "cases": 50, + "passed": 35, + "accuracy": 0.7, + "weighted_points": 58.85, + "possible_weighted_points": 87.17500000000001, + "weighted_accuracy": 0.6750788643533122, + "elo": 1084, + "elo_unrounded": 1084.314352251316 + }, + "world_knowledge": { + "cases": 50, + "passed": 38, + "accuracy": 0.76, + "weighted_points": 66.0, + "possible_weighted_points": 87.72500000000001, + "weighted_accuracy": 0.7523510971786833, + "elo": 1151, + "elo_unrounded": 1150.5543694613593 + }, + "context_tracking": { + "cases": 50, + "passed": 26, + "accuracy": 0.52, + "weighted_points": 47.699999999999996, + "possible_weighted_points": 94.19999999999999, + "weighted_accuracy": 0.5063694267515924, + "elo": 1002, + "elo_unrounded": 1002.410277417023 + }, + "quantitative": { + "cases": 50, + "passed": 17, + "accuracy": 0.34, + "weighted_points": 34.125, + "possible_weighted_points": 103.025, + "weighted_accuracy": 0.3312302839116719, + "elo": 923, + "elo_unrounded": 922.9160309715924 + }, + "logical_reasoning": { + "cases": 50, + "passed": 25, + "accuracy": 0.5, + "weighted_points": 51.300000000000004, + "possible_weighted_points": 107.66250000000001, + "weighted_accuracy": 0.47648902821316613, + "elo": 1082, + "elo_unrounded": 1081.6919799942607 + }, + "code_completion": { + "cases": 50, + "passed": 43, + "accuracy": 0.86, + "weighted_points": 93.8, + "possible_weighted_points": 109.89999999999999, + "weighted_accuracy": 0.8535031847133758, + "elo": 1403, + "elo_unrounded": 1402.753907086344 + } + }, + "difficulties": { + "easy": { + "cases": 117, + "passed": 83, + "accuracy": 0.7094017094017094, + "weighted_points": 98.60000000000001, + "possible_weighted_points": 141.2, + "weighted_accuracy": 0.6983002832861192, + "elo": 1016, + "elo_unrounded": 1016.3133811577516 + }, + "medium": { + "cases": 117, + "passed": 77, + "accuracy": 0.6581196581196581, + "weighted_points": 137.925, + "possible_weighted_points": 211.875, + "weighted_accuracy": 0.6509734513274337, + "elo": 1077, + "elo_unrounded": 1076.9044441063706 + }, + "hard": { + "cases": 116, + "passed": 71, + "accuracy": 0.6120689655172413, + "weighted_points": 189.0, + "possible_weighted_points": 315.1125, + "weighted_accuracy": 0.599785790789004, + "elo": 1187, + "elo_unrounded": 1186.5309969349169 + } + }, + "context_truncations": 0 + }, + "results": [ + { + "id": "language_completion-001", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Dark clouds gathered above the park, so Mira opened", + "continuations": [ + " a sandwich loudly.", + " the sunlight.", + " her umbrella.", + " the calendar yesterday." + ], + "tags": [ + "coherence" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -29.258682250976562, + "mean_logprob": -4.876447041829427, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -14.473627090454102, + "mean_logprob": -4.8245423634847, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -8.218729019165039, + "mean_logprob": -1.3697881698608398, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -22.170177459716797, + "mean_logprob": -5.542544364929199, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 3.4547541936238604, + "context_truncated": false, + "elapsed_seconds": 0.9227663130004657 + }, + { + "id": "language_completion-002", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "The librarian asked everyone to speak", + "continuations": [ + " brightly.", + " quietly.", + " woodenly.", + " tomorrow." + ], + "tags": [ + "adverb" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -15.653277397155762, + "mean_logprob": -5.217759132385254, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.187465190887451, + "mean_logprob": -2.395821730295817, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -20.17020034790039, + "mean_logprob": -6.723400115966797, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -11.879611015319824, + "mean_logprob": -5.939805507659912, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 2.821937402089437, + "context_truncated": false, + "elapsed_seconds": 0.051079696999295265 + }, + { + "id": "language_completion-003", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 0.0, + "item_elo": 750, + "context": "The brass key fit perfectly into the", + "continuations": [ + " pillow.", + " thunder.", + " lock.", + " soup." + ], + "tags": [ + "collocation" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -10.946988105773926, + "mean_logprob": -3.648996035257975, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -18.382553100585938, + "mean_logprob": -6.1275177001953125, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -7.96212100982666, + "mean_logprob": -3.98106050491333, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -14.014673233032227, + "mean_logprob": -7.007336616516113, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.33206446965535497, + "context_truncated": false, + "elapsed_seconds": 0.043018684000344365 + }, + { + "id": "language_completion-004", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "After kneading the dough, the baker placed it in the", + "continuations": [ + " envelope.", + " freezer to read.", + " bicycle.", + " oven." + ], + "tags": [ + "coherence" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -16.552427291870117, + "mean_logprob": -8.276213645935059, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -25.37650489807129, + "mean_logprob": -5.075300979614258, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -25.005922317504883, + "mean_logprob": -12.502961158752441, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -2.8407983779907227, + "mean_logprob": -1.4203991889953613, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 3.6549017906188963, + "context_truncated": false, + "elapsed_seconds": 0.05098429600002419 + }, + { + "id": "language_completion-005", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Leo was exhausted after the hike and went straight to", + "continuations": [ + " bed.", + " the trumpet.", + " blue.", + " calculate." + ], + "tags": [ + "coherence" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -2.9319279193878174, + "mean_logprob": -1.4659639596939087, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -15.567556381225586, + "mean_logprob": -3.1135112762451174, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -10.717717170715332, + "mean_logprob": -5.358858585357666, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -17.180051803588867, + "mean_logprob": -8.590025901794434, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 1.6475473165512087, + "context_truncated": false, + "elapsed_seconds": 0.041541919999872334 + }, + { + "id": "language_completion-006", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "The ice cube disappeared because it had", + "continuations": [ + " flown north.", + " melted.", + " written a letter.", + " become louder." + ], + "tags": [ + "verb" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -15.799678802490234, + "mean_logprob": -3.9499197006225586, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -3.6080100536346436, + "mean_logprob": -1.2026700178782146, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -17.65917205810547, + "mean_logprob": -4.414793014526367, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -17.602617263793945, + "mean_logprob": -3.520523452758789, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 2.3178534348805746, + "context_truncated": false, + "elapsed_seconds": 0.0459696409998287 + }, + { + "id": "language_completion-007", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "One child waited outside; soon, three more", + "continuations": [ + " childs arrived.", + " childes arrived.", + " children arrived.", + " child arriveds." + ], + "tags": [ + "grammar" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -14.999372482299805, + "mean_logprob": -3.749843120574951, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -24.83102798461914, + "mean_logprob": -6.207756996154785, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -6.094472885131836, + "mean_logprob": -2.031490961710612, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -26.04423713684082, + "mean_logprob": -6.511059284210205, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 1.7183521588643393, + "context_truncated": false, + "elapsed_seconds": 0.04569892300060019 + }, + { + "id": "language_completion-008", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Yesterday, Priya walked to the station and", + "continuations": [ + " waits for tomorrow.", + " waiting there now.", + " will walked home.", + " bought a ticket." + ], + "tags": [ + "tense" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -27.789226531982422, + "mean_logprob": -5.5578453063964846, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -24.613021850585938, + "mean_logprob": -6.153255462646484, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -27.470468521118164, + "mean_logprob": -6.867617130279541, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -9.572569847106934, + "mean_logprob": -2.3931424617767334, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 3.164702844619751, + "context_truncated": false, + "elapsed_seconds": 0.043610005001028185 + }, + { + "id": "language_completion-009", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "The cup is empty, but the pitcher is", + "continuations": [ + " full.", + " emptier than silence.", + " running yesterday.", + " carefully." + ], + "tags": [ + "contrast" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -0.5352787971496582, + "mean_logprob": -0.2676393985748291, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -30.967300415039062, + "mean_logprob": -5.161216735839844, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -23.753543853759766, + "mean_logprob": -7.917847951253255, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -22.881458282470703, + "mean_logprob": -11.440729141235352, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 4.893577337265015, + "context_truncated": false, + "elapsed_seconds": 0.04228099199826829 + }, + { + "id": "language_completion-010", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Neither the teacher nor the students", + "continuations": [ + " was arrives.", + " were ready.", + " is readiness.", + " be readying." + ], + "tags": [ + "agreement" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -25.68792152404785, + "mean_logprob": -6.421980381011963, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -9.509971618652344, + "mean_logprob": -3.1699905395507812, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -28.661731719970703, + "mean_logprob": -7.165432929992676, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -28.500198364257812, + "mean_logprob": -7.125049591064453, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 3.2519898414611816, + "context_truncated": false, + "elapsed_seconds": 0.05342107900105475 + }, + { + "id": "language_completion-011", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Nora packed an apple, a bottle of water, and", + "continuations": [ + " because.", + " beneath.", + " a sandwich.", + " although." + ], + "tags": [ + "parallelism" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -21.972896575927734, + "mean_logprob": -10.986448287963867, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -21.476001739501953, + "mean_logprob": -10.738000869750977, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -7.899407386779785, + "mean_logprob": -1.9748518466949463, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -24.667869567871094, + "mean_logprob": -12.333934783935547, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 8.76314902305603, + "context_truncated": false, + "elapsed_seconds": 0.03638890000001993 + }, + { + "id": "language_completion-012", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "A robin is a bird, and a salmon is", + "continuations": [ + " beneath flying.", + " an quickly.", + " yesterday's wing.", + " a fish." + ], + "tags": [ + "syntax" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -33.78597640991211, + "mean_logprob": -11.26199213663737, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -33.265933990478516, + "mean_logprob": -11.088644663492838, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -36.72492980957031, + "mean_logprob": -7.3449859619140625, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -0.36596569418907166, + "mean_logprob": -0.12198856472969055, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 7.222997397184372, + "context_truncated": false, + "elapsed_seconds": 0.03459843299970089 + }, + { + "id": "language_completion-013", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "She adopted a puppy from", + "continuations": [ + " an animal shelter.", + " shelter animal an.", + " a sheltering quickly.", + " the adopted because." + ], + "tags": [ + "article" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -7.865048408508301, + "mean_logprob": -1.9662621021270752, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -32.348663330078125, + "mean_logprob": -8.087165832519531, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -33.793785095214844, + "mean_logprob": -6.758757019042969, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -30.407835006713867, + "mean_logprob": -7.601958751678467, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 4.792494916915894, + "context_truncated": false, + "elapsed_seconds": 0.03482184299900837 + }, + { + "id": "language_completion-014", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Daniel thanked Rosa because", + "continuations": [ + " him had helped she.", + " she had helped him.", + " her helped himself.", + " they helps he." + ], + "tags": [ + "pronouns" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -32.18361282348633, + "mean_logprob": -6.4367225646972654, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -11.391644477844238, + "mean_logprob": -2.2783288955688477, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -28.635631561279297, + "mean_logprob": -7.158907890319824, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -32.779876708984375, + "mean_logprob": -8.194969177246094, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 4.158393669128418, + "context_truncated": false, + "elapsed_seconds": 0.03904251700078021 + }, + { + "id": "language_completion-015", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "This suitcase is heavy, but that one is even", + "continuations": [ + " most heavy.", + " heavily one.", + " heavier.", + " more heavier." + ], + "tags": [ + "comparative" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -16.571857452392578, + "mean_logprob": -5.523952484130859, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -23.978748321533203, + "mean_logprob": -7.992916107177734, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -1.3102918863296509, + "mean_logprob": -0.4367639621098836, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -8.343090057373047, + "mean_logprob": -2.0857725143432617, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 1.6490085522333782, + "context_truncated": false, + "elapsed_seconds": 0.03316458200060879 + }, + { + "id": "language_completion-016", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Monday comes immediately after", + "continuations": [ + " Wednesday.", + " evening.", + " January.", + " Sunday." + ], + "tags": [ + "sequence" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -9.539777755737305, + "mean_logprob": -4.769888877868652, + "total_sequence_tokens": 7 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.153751373291016, + "mean_logprob": -5.576875686645508, + "total_sequence_tokens": 7 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -11.202709197998047, + "mean_logprob": -5.601354598999023, + "total_sequence_tokens": 7 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -7.936887264251709, + "mean_logprob": -3.9684436321258545, + "total_sequence_tokens": 7 + } + ], + "best_vs_second_margin": 0.8014452457427979, + "context_truncated": false, + "elapsed_seconds": 0.04127686799984076 + }, + { + "id": "language_completion-017", + "category": "language_completion", + "difficulty": "easy", + "category_weight": 1.0, + "difficulty_weight": 1.0, + "point_weight": 1.0, + "earned_points": 1.0, + "item_elo": 750, + "context": "Although the road was narrow, the driver remained", + "continuations": [ + " careful.", + " width.", + " carefully road.", + " narrowness." + ], + "tags": [ + "grammar" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -7.293829917907715, + "mean_logprob": -3.6469149589538574, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.16122055053711, + "mean_logprob": -10.080610275268555, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -23.18958854675293, + "mean_logprob": -7.7298628489176435, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -20.18323516845703, + "mean_logprob": -6.727745056152344, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 3.0808300971984863, + "context_truncated": false, + "elapsed_seconds": 0.033489133000330185 + }, + { + "id": "language_completion-018", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 0.0, + "item_elo": 850, + "context": "The forecast promised sunshine. Nevertheless, we packed", + "continuations": [ + " the forecast itself.", + " rain jackets.", + " a darker noon.", + " sunshine into boxes." + ], + "tags": [ + "discourse" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -19.17872428894043, + "mean_logprob": -4.794681072235107, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -16.765783309936523, + "mean_logprob": -4.191445827484131, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -29.223602294921875, + "mean_logprob": -4.8706003824869795, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -22.123672485351562, + "mean_logprob": -3.6872787475585938, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.5041670799255371, + "context_truncated": false, + "elapsed_seconds": 0.03606204700008675 + }, + { + "id": "language_completion-019", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Amir revised the report twice before submitting it, thereby", + "continuations": [ + " forgetting what reports are.", + " submitting before revising.", + " reducing the chance of errors.", + " making twice become three." + ], + "tags": [ + "discourse" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -40.61179733276367, + "mean_logprob": -6.768632888793945, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -27.6369686126709, + "mean_logprob": -4.606161435445149, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -12.523467063903809, + "mean_logprob": -2.0872445106506348, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -44.723777770996094, + "mean_logprob": -8.944755554199219, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 2.5189169247945147, + "context_truncated": false, + "elapsed_seconds": 0.034903477999250754 + }, + { + "id": "language_completion-020", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The museum closes at six. We arrived at five forty-five, leaving us", + "continuations": [ + " closed before arrival.", + " six museums to visit.", + " yesterday instead.", + " only fifteen minutes." + ], + "tags": [ + "coherence" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -27.271808624267578, + "mean_logprob": -6.8179521560668945, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -19.100353240966797, + "mean_logprob": -3.8200706481933593, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -22.578886032104492, + "mean_logprob": -7.526295344034831, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -13.414216995239258, + "mean_logprob": -3.3535542488098145, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.46651639938354483, + "context_truncated": false, + "elapsed_seconds": 0.03929743299886468 + }, + { + "id": "language_completion-021", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The committee postponed the vote until every member had", + "continuations": [ + " reviewed the proposal.", + " been proposal by voting.", + " postponed every committee.", + " review the voted." + ], + "tags": [ + "aspect" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -10.851517677307129, + "mean_logprob": -2.7128794193267822, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -34.83161544799805, + "mean_logprob": -6.9663230895996096, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -32.971126556396484, + "mean_logprob": -5.495187759399414, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -32.07133483886719, + "mean_logprob": -8.017833709716797, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 2.782308340072632, + "context_truncated": false, + "elapsed_seconds": 0.030963249000706128 + }, + { + "id": "language_completion-022", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Had Elena known the bridge was closed, she", + "continuations": [ + " takes that route anyway.", + " would have taken another route.", + " will know it tomorrow yesterday.", + " had closes the bridge." + ], + "tags": [ + "conditional" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -27.9699649810791, + "mean_logprob": -5.59399299621582, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -18.3637752532959, + "mean_logprob": -3.06062920888265, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -39.02067184448242, + "mean_logprob": -6.503445307413737, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -29.636816024780273, + "mean_logprob": -4.939469337463379, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.878840128580729, + "context_truncated": false, + "elapsed_seconds": 0.03254597900013323 + }, + { + "id": "language_completion-023", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The novel was praised not only for its plot but also for", + "continuations": [ + " it was long.", + " plotting beyond.", + " its vivid characters.", + " praise was also." + ], + "tags": [ + "parallelism" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -33.65026092529297, + "mean_logprob": -8.412565231323242, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -31.721717834472656, + "mean_logprob": -7.930429458618164, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -13.388650894165039, + "mean_logprob": -3.3471627235412598, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -50.499691009521484, + "mean_logprob": -10.099938201904298, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 4.583266735076904, + "context_truncated": false, + "elapsed_seconds": 0.0373120789990935 + }, + { + "id": "language_completion-024", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Because the instructions were ambiguous, two technicians interpreted them", + "continuations": [ + " with identical ambiguity only.", + " before they were written.", + " into a screwdriver.", + " differently." + ], + "tags": [ + "semantics" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -30.603696823120117, + "mean_logprob": -6.120739364624024, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -17.289165496826172, + "mean_logprob": -3.457833099365234, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -19.69405174255371, + "mean_logprob": -3.282341957092285, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -2.138988971710205, + "mean_logprob": -1.0694944858551025, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 2.2128474712371826, + "context_truncated": false, + "elapsed_seconds": 0.02990865999890957 + }, + { + "id": "language_completion-025", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Sofia lowered her voice so that she would not", + "continuations": [ + " wake the baby.", + " sleeping the quiet.", + " lower the voice's color.", + " have loudly whisper." + ], + "tags": [ + "purpose" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -16.679588317871094, + "mean_logprob": -4.169897079467773, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -38.57061767578125, + "mean_logprob": -9.642654418945312, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -33.39238739013672, + "mean_logprob": -4.770341055733817, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -32.384063720703125, + "mean_logprob": -5.3973439534505205, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.6004439762660434, + "context_truncated": false, + "elapsed_seconds": 0.03863776300022437 + }, + { + "id": "language_completion-026", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The first experiment failed; the second, by contrast,", + "continuations": [ + " was experiment first.", + " produced reliable results.", + " contrasted before being second.", + " failing reliably." + ], + "tags": [ + "discourse" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -27.998268127441406, + "mean_logprob": -6.999567031860352, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -9.488460540771484, + "mean_logprob": -2.372115135192871, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -40.71430206298828, + "mean_logprob": -6.785717010498047, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -21.225833892822266, + "mean_logprob": -5.306458473205566, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 2.9343433380126953, + "context_truncated": false, + "elapsed_seconds": 0.028666005999184563 + }, + { + "id": "language_completion-027", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Every applicant must attach a resume, regardless of", + "continuations": [ + " attaching is a verb.", + " resumes being paper.", + " prior experience.", + " every regardless." + ], + "tags": [ + "collocation" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -33.530799865722656, + "mean_logprob": -5.588466644287109, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -28.748760223388672, + "mean_logprob": -5.749752044677734, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -9.654231071472168, + "mean_logprob": -3.218077023824056, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -31.807788848876953, + "mean_logprob": -10.602596282958984, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 2.3703896204630532, + "context_truncated": false, + "elapsed_seconds": 0.02823326300131157 + }, + { + "id": "language_completion-028", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The recipe calls for fresh basil; dried basil may be used", + "continuations": [ + " before fresh basil exists.", + " to call the recipe.", + " as a plate.", + " as a substitute." + ], + "tags": [ + "coherence" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -19.658309936523438, + "mean_logprob": -3.2763849894205728, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -22.95904541015625, + "mean_logprob": -4.59180908203125, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -20.08816146850586, + "mean_logprob": -5.022040367126465, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -4.103679180145264, + "mean_logprob": -1.025919795036316, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 2.250465194384257, + "context_truncated": false, + "elapsed_seconds": 0.03386163600043801 + }, + { + "id": "language_completion-029", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Far from being discouraged by the criticism, Mei became", + "continuations": [ + " more determined.", + " criticism itself.", + " far from becoming.", + " determine yesterday." + ], + "tags": [ + "idiom" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -8.815675735473633, + "mean_logprob": -2.938558578491211, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -23.545949935913086, + "mean_logprob": -7.848649978637695, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -30.538150787353516, + "mean_logprob": -7.634537696838379, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -30.116004943847656, + "mean_logprob": -10.038668314615885, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 4.695979118347168, + "context_truncated": false, + "elapsed_seconds": 0.02743094600009499 + }, + { + "id": "language_completion-030", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The contract is valid provided that both parties", + "continuations": [ + " had validity tomorrow.", + " sign it.", + " providing a party.", + " signs them." + ], + "tags": [ + "conditional" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -35.413047790527344, + "mean_logprob": -8.853261947631836, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -8.452535629272461, + "mean_logprob": -2.817511876424154, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -28.993125915527344, + "mean_logprob": -7.248281478881836, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -19.822309494018555, + "mean_logprob": -6.607436498006185, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 3.789924621582031, + "context_truncated": false, + "elapsed_seconds": 0.028958492999663576 + }, + { + "id": "language_completion-031", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "No sooner had the bell rung than the students", + "continuations": [ + " have ringing bells.", + " sooner the bell.", + " left the classroom.", + " had leave tomorrow." + ], + "tags": [ + "inversion" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -27.051218032836914, + "mean_logprob": -4.508536338806152, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -22.70492935180664, + "mean_logprob": -4.540985870361328, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -8.72513484954834, + "mean_logprob": -2.181283712387085, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -21.85160255432129, + "mean_logprob": -5.462900638580322, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 2.3272526264190674, + "context_truncated": false, + "elapsed_seconds": 0.03914885999984108 + }, + { + "id": "language_completion-032", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The evidence was incomplete. Consequently, the investigators", + "continuations": [ + " completed evidence by definition.", + " investigated before consequence.", + " were the evidence.", + " withheld judgment." + ], + "tags": [ + "discourse" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -37.00300598144531, + "mean_logprob": -7.400601196289062, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -32.91265869140625, + "mean_logprob": -8.228164672851562, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -31.54528045654297, + "mean_logprob": -7.886320114135742, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -17.593650817871094, + "mean_logprob": -4.398412704467773, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 3.002188491821289, + "context_truncated": false, + "elapsed_seconds": 0.033153643998957705 + }, + { + "id": "language_completion-033", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "Walking along the shore, I noticed", + "continuations": [ + " a trail of fresh footprints.", + " the shore walked me.", + " noticing had walked.", + " along was a footprint." + ], + "tags": [ + "modifier" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -24.315420150756836, + "mean_logprob": -3.4736314501081194, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -27.378658294677734, + "mean_logprob": -5.475731658935547, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -39.87729263305664, + "mean_logprob": -7.9754585266113285, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -41.18281936645508, + "mean_logprob": -6.863803227742513, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 2.0021002088274273, + "context_truncated": false, + "elapsed_seconds": 0.03778556900033436 + }, + { + "id": "language_completion-034", + "category": "language_completion", + "difficulty": "medium", + "category_weight": 1.0, + "difficulty_weight": 1.5, + "point_weight": 1.5, + "earned_points": 1.5, + "item_elo": 850, + "context": "The orchestra rehearsed the difficult passage repeatedly until it sounded", + "continuations": [ + " passage by difficulty.", + " effortless.", + " rehearsing the sound.", + " repeatably orchestra." + ], + "tags": [ + "collocation" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -32.8645133972168, + "mean_logprob": -8.2161283493042, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -5.816623687744141, + "mean_logprob": -1.9388745625813801, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -31.051162719726562, + "mean_logprob": -4.435880388532366, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -39.944427490234375, + "mean_logprob": -6.6574045817057295, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 2.4970058259509864, + "context_truncated": false, + "elapsed_seconds": 0.04088619899994228 + }, + { + "id": "language_completion-035", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "What the witness omitted from her statement proved", + "continuations": [ + " that omission witnessed.", + " proving had omitted.", + " more significant than what she included.", + " to include more omittedly." + ], + "tags": [ + "syntax" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -31.159378051757812, + "mean_logprob": -6.231875610351563, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -33.38984680175781, + "mean_logprob": -8.347461700439453, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -23.858367919921875, + "mean_logprob": -3.4083382742745534, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -43.83302307128906, + "mean_logprob": -7.305503845214844, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 2.8235373360770093, + "context_truncated": false, + "elapsed_seconds": 0.05545987199911906 + }, + { + "id": "language_completion-036", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Were the assumptions underlying the estimate to change, the conclusion would", + "continuations": [ + " have assumptions beneath it.", + " changing was estimated.", + " underlie tomorrow.", + " need to be reconsidered." + ], + "tags": [ + "conditional" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -28.43362045288086, + "mean_logprob": -5.686724090576172, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -36.489532470703125, + "mean_logprob": -9.122383117675781, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -29.69447135925293, + "mean_logprob": -7.423617839813232, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -9.104352951049805, + "mean_logprob": -1.300621850149972, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 4.3861022404262, + "context_truncated": false, + "elapsed_seconds": 0.029180166000514873 + }, + { + "id": "language_completion-037", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The policy, while defensible in principle, is difficult to", + "continuations": [ + " implement consistently.", + " principle the defense.", + " difficult implementation was.", + " consistency defending." + ], + "tags": [ + "concession" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -6.661409378051758, + "mean_logprob": -2.2204697926839194, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.245487213134766, + "mean_logprob": -8.811371803283691, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -46.78346633911133, + "mean_logprob": -11.695866584777832, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -36.44639587402344, + "mean_logprob": -9.11159896850586, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 6.5909020105997715, + "context_truncated": false, + "elapsed_seconds": 0.027174740000191377 + }, + { + "id": "language_completion-038", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Only after the archive was digitized did researchers", + "continuations": [ + " digitized only before.", + " discover the missing correspondence.", + " archive the discovery did.", + " correspondingly research." + ], + "tags": [ + "inversion" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -35.61037826538086, + "mean_logprob": -7.1220756530761715, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -25.180889129638672, + "mean_logprob": -5.0361778259277346, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -38.086387634277344, + "mean_logprob": -7.617277526855469, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -29.758289337158203, + "mean_logprob": -7.439572334289551, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 2.085897827148437, + "context_truncated": false, + "elapsed_seconds": 0.030827698999928543 + }, + { + "id": "language_completion-039", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The editor objected less to the claim itself than to", + "continuations": [ + " claims having objections.", + " itself being less.", + " the certainty with which it was stated.", + " stating an editor." + ], + "tags": [ + "comparison" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -23.438323974609375, + "mean_logprob": -3.9063873291015625, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -33.21238708496094, + "mean_logprob": -8.303096771240234, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -24.74079132080078, + "mean_logprob": -2.748976813422309, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -31.99425506591797, + "mean_logprob": -6.398851013183593, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.1574105156792536, + "context_truncated": false, + "elapsed_seconds": 0.026268130000971723 + }, + { + "id": "language_completion-040", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "So subtle was the change in tone that few listeners", + "continuations": [ + " made tone into change.", + " subtly were few.", + " had listened tomorrow.", + " noticed it." + ], + "tags": [ + "inversion" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -36.30894470214844, + "mean_logprob": -7.261788940429687, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -43.557823181152344, + "mean_logprob": -8.711564636230468, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -30.78145980834961, + "mean_logprob": -6.156291961669922, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -13.248579025268555, + "mean_logprob": -4.416193008422852, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 1.7400989532470703, + "context_truncated": false, + "elapsed_seconds": 0.026592488000460435 + }, + { + "id": "language_completion-041", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The proposal is neither as costly as its critics allege nor as simple as its supporters", + "continuations": [ + " suggest.", + " allegation was cost.", + " supporting simply.", + " critics and costly." + ], + "tags": [ + "parallelism" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -4.625497817993164, + "mean_logprob": -2.312748908996582, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -36.28232192993164, + "mean_logprob": -7.2564643859863285, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -27.830577850341797, + "mean_logprob": -9.276859283447266, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -37.068687438964844, + "mean_logprob": -9.267171859741211, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 4.9437154769897464, + "context_truncated": false, + "elapsed_seconds": 0.0277293990002363 + }, + { + "id": "language_completion-042", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Whatever reservations the panel may have had, they were not sufficient to", + "continuations": [ + " reserve a panel seat.", + " block the appointment.", + " appoint sufficient reservations.", + " having been whatever." + ], + "tags": [ + "concession" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -20.758148193359375, + "mean_logprob": -4.151629638671875, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -14.865694046020508, + "mean_logprob": -3.716423511505127, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -24.439010620117188, + "mean_logprob": -4.8878021240234375, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -34.549068450927734, + "mean_logprob": -8.637267112731934, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.4352061271667482, + "context_truncated": false, + "elapsed_seconds": 0.05680329200004053 + }, + { + "id": "language_completion-043", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Had the data been interpreted in isolation, the apparent trend might have", + "continuations": [ + " isolated its data.", + " been interpreting apparent.", + " seemed more convincing.", + " trended the isolation." + ], + "tags": [ + "counterfactual" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -24.237958908081055, + "mean_logprob": -6.059489727020264, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -35.368247985839844, + "mean_logprob": -7.073649597167969, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -17.2215518951416, + "mean_logprob": -2.870258649190267, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -35.14228820800781, + "mean_logprob": -7.0284576416015625, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 3.1892310778299966, + "context_truncated": false, + "elapsed_seconds": 0.059677686000213725 + }, + { + "id": "language_completion-044", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The mayor's apology did little to allay concerns that the inquiry had been", + "continuations": [ + " mayoral and apologetic.", + " concerned into inquiry.", + " doing little apology.", + " deliberately delayed." + ], + "tags": [ + "collocation" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -28.935810089111328, + "mean_logprob": -4.1336871555873325, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -41.75049591064453, + "mean_logprob": -10.437623977661133, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -31.012474060058594, + "mean_logprob": -6.202494812011719, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -6.5513129234313965, + "mean_logprob": -1.6378282308578491, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 2.4958589247294833, + "context_truncated": false, + "elapsed_seconds": 0.04653508400042483 + }, + { + "id": "language_completion-045", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Not until the final chapter does the significance of the opening scene become", + "continuations": [ + " fully apparent.", + " final before opening.", + " chaptered significantly.", + " opening does final." + ], + "tags": [ + "inversion" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -7.648927211761475, + "mean_logprob": -2.5496424039204917, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -25.284252166748047, + "mean_logprob": -6.321063041687012, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.81638717651367, + "mean_logprob": -9.454096794128418, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -36.81341552734375, + "mean_logprob": -9.203353881835938, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 3.77142063776652, + "context_truncated": false, + "elapsed_seconds": 0.027118156998767518 + }, + { + "id": "language_completion-046", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 0.0, + "item_elo": 1000, + "context": "The theory's appeal lies not in its novelty but in its ability to", + "continuations": [ + " appeal theoretically.", + " unify previously separate observations.", + " become novel before theory.", + " observe its ability." + ], + "tags": [ + "semantics" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -18.210508346557617, + "mean_logprob": -3.6421016693115233, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -25.781158447265625, + "mean_logprob": -4.2968597412109375, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -40.01434326171875, + "mean_logprob": -8.00286865234375, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -31.227251052856445, + "mean_logprob": -7.806812763214111, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.6547580718994142, + "context_truncated": false, + "elapsed_seconds": 0.03853663200061419 + }, + { + "id": "language_completion-047", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Much as the negotiators wanted an agreement, neither side was prepared to", + "continuations": [ + " negotiate wanting.", + " agreement much as.", + " concede the central point.", + " prepare neither side." + ], + "tags": [ + "concession" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -23.181316375732422, + "mean_logprob": -4.636263275146485, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -35.157752990722656, + "mean_logprob": -8.789438247680664, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -22.38667106628418, + "mean_logprob": -3.1980958666120256, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -27.910926818847656, + "mean_logprob": -6.977731704711914, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 1.4381674085344591, + "context_truncated": false, + "elapsed_seconds": 0.027152167000167537 + }, + { + "id": "language_completion-048", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "Whether the discrepancy reflects measurement error or a genuine anomaly remains", + "continuations": [ + " measuring a reflection.", + " discrepant in error.", + " anomalously genuine.", + " uncertain." + ], + "tags": [ + "syntax" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -34.6393928527832, + "mean_logprob": -8.6598482131958, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -32.52997970581055, + "mean_logprob": -5.421663284301758, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -37.232521057128906, + "mean_logprob": -7.446504211425781, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -2.806727170944214, + "mean_logprob": -1.403363585472107, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 4.018299698829651, + "context_truncated": false, + "elapsed_seconds": 0.02666539100027876 + }, + { + "id": "language_completion-049", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "The ruling established a precedent whose consequences would not become clear", + "continuations": [ + " for several years.", + " precedent had ruled.", + " consequences establishing.", + " clearly whose." + ], + "tags": [ + "relative_clause" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -9.659148216247559, + "mean_logprob": -2.4147870540618896, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -32.767478942871094, + "mean_logprob": -6.553495788574219, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -32.46292495727539, + "mean_logprob": -10.820974985758463, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -32.78173828125, + "mean_logprob": -10.92724609375, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 4.1387087345123295, + "context_truncated": false, + "elapsed_seconds": 0.026745708999442286 + }, + { + "id": "language_completion-050", + "category": "language_completion", + "difficulty": "hard", + "category_weight": 1.0, + "difficulty_weight": 2.25, + "point_weight": 2.25, + "earned_points": 2.25, + "item_elo": 1000, + "context": "To attribute the recovery solely to the new policy would be to", + "continuations": [ + " recover sole attribution.", + " ignore several contributing factors.", + " policy the recovery solely.", + " contribute an attribution." + ], + "tags": [ + "idiom" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -28.278736114501953, + "mean_logprob": -5.6557472229003904, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -23.966026306152344, + "mean_logprob": -4.793205261230469, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -29.296300888061523, + "mean_logprob": -5.859260177612304, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -25.130556106567383, + "mean_logprob": -5.026111221313476, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.23290596008300746, + "context_truncated": false, + "elapsed_seconds": 0.027140629999848898 + }, + { + "id": "commonsense-001", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "A glass slipped from the table onto a tile floor. It most likely", + "continuations": [ + " grew roots.", + " floated upward.", + " shattered.", + " became warmer than fire." + ], + "tags": [ + "physical" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -17.35636329650879, + "mean_logprob": -5.785454432169597, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -17.766342163085938, + "mean_logprob": -4.441585540771484, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -8.128620147705078, + "mean_logprob": -2.0321550369262695, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -26.523273468017578, + "mean_logprob": -5.304654693603515, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 2.409430503845215, + "context_truncated": false, + "elapsed_seconds": 0.07991852200029825 + }, + { + "id": "commonsense-002", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Jules wanted to mail a letter, so he put a stamp on the", + "continuations": [ + " refrigerator.", + " envelope.", + " sidewalk.", + " spoon." + ], + "tags": [ + "everyday" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -10.811249732971191, + "mean_logprob": -2.702812433242798, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -0.9752203226089478, + "mean_logprob": -0.4876101613044739, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -11.075915336608887, + "mean_logprob": -2.7689788341522217, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -11.012920379638672, + "mean_logprob": -3.670973459879557, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 2.215202271938324, + "context_truncated": false, + "elapsed_seconds": 0.0346436170002562 + }, + { + "id": "commonsense-003", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The room was dark until Ana flipped the wall switch and the", + "continuations": [ + " carpet vanished.", + " window froze.", + " lights came on.", + " ceiling became wet." + ], + "tags": [ + "causality" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -19.102373123168945, + "mean_logprob": -3.820474624633789, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -11.570735931396484, + "mean_logprob": -2.892683982849121, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -7.960273265838623, + "mean_logprob": -1.9900683164596558, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -17.106468200683594, + "mean_logprob": -4.276617050170898, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.9026156663894653, + "context_truncated": false, + "elapsed_seconds": 0.028131573999417014 + }, + { + "id": "commonsense-004", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Before slicing a loaf of bread, most people reach for", + "continuations": [ + " a paintbrush.", + " a blanket.", + " a shoe.", + " a knife." + ], + "tags": [ + "tool_use" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -14.79000186920166, + "mean_logprob": -3.697500467300415, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -14.01634693145752, + "mean_logprob": -3.50408673286438, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -10.878457069396973, + "mean_logprob": -2.719614267349243, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -2.3763246536254883, + "mean_logprob": -0.5940811634063721, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 2.125533103942871, + "context_truncated": false, + "elapsed_seconds": 0.031615363999662804 + }, + { + "id": "commonsense-005", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "If a houseplant's soil is very dry, the plant probably needs", + "continuations": [ + " water.", + " a password.", + " louder music.", + " a winter coat." + ], + "tags": [ + "everyday" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -3.1428794860839844, + "mean_logprob": -1.5714397430419922, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -22.261730194091797, + "mean_logprob": -7.420576731363933, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -34.00652313232422, + "mean_logprob": -6.801304626464844, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -15.291974067687988, + "mean_logprob": -3.822993516921997, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 2.251553773880005, + "context_truncated": false, + "elapsed_seconds": 0.06647472599979665 + }, + { + "id": "commonsense-006", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Omar put leftovers in the refrigerator to keep them", + "continuations": [ + " invisible.", + " cold.", + " expensive.", + " rectangular." + ], + "tags": [ + "everyday" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -14.542591094970703, + "mean_logprob": -7.271295547485352, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -3.701167106628418, + "mean_logprob": -1.850583553314209, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -16.922225952148438, + "mean_logprob": -8.461112976074219, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -17.653610229492188, + "mean_logprob": -8.826805114746094, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 5.420711994171143, + "context_truncated": false, + "elapsed_seconds": 0.02647093199993833 + }, + { + "id": "commonsense-007", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Someone standing outside in heavy rain without shelter will probably get", + "continuations": [ + " taller.", + " younger.", + " wet.", + " magnetic." + ], + "tags": [ + "physical" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -11.499820709228516, + "mean_logprob": -3.8332735697428384, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.593626976013184, + "mean_logprob": -6.796813488006592, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -5.4803338050842285, + "mean_logprob": -2.7401669025421143, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -16.12941551208496, + "mean_logprob": -8.06470775604248, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 1.0931066672007241, + "context_truncated": false, + "elapsed_seconds": 0.028404538999893703 + }, + { + "id": "commonsense-008", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "A cyclist approaching a red traffic light should", + "continuations": [ + " accelerate through every lane.", + " close both eyes.", + " remove the wheels.", + " stop." + ], + "tags": [ + "safety" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -21.939393997192383, + "mean_logprob": -3.1341991424560547, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -18.764877319335938, + "mean_logprob": -4.691219329833984, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -22.600740432739258, + "mean_logprob": -5.6501851081848145, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -4.850698471069336, + "mean_logprob": -2.425349235534668, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.7088499069213867, + "context_truncated": false, + "elapsed_seconds": 0.03384869600085949 + }, + { + "id": "commonsense-009", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "To find out whether someone has a fever, a nurse commonly uses", + "continuations": [ + " a thermometer.", + " a ruler.", + " a compass.", + " a stapler." + ], + "tags": [ + "tool_use" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -7.429974555969238, + "mean_logprob": -1.8574936389923096, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -12.03149700164795, + "mean_logprob": -4.010499000549316, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -12.593362808227539, + "mean_logprob": -4.197787602742513, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -16.09111976623535, + "mean_logprob": -3.2182239532470702, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 1.3607303142547607, + "context_truncated": false, + "elapsed_seconds": 0.030042739999771584 + }, + { + "id": "commonsense-010", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "When the phone battery reaches zero, the phone usually", + "continuations": [ + " doubles in size.", + " turns off.", + " prints a book.", + " becomes waterproof." + ], + "tags": [ + "everyday" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -10.272951126098633, + "mean_logprob": -2.0545902252197266, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -6.171103000640869, + "mean_logprob": -2.0570343335469565, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -26.03506851196289, + "mean_logprob": -6.508767127990723, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -16.60114288330078, + "mean_logprob": -4.150285720825195, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.0024441083272299657, + "context_truncated": false, + "elapsed_seconds": 0.03187223499844549 + }, + { + "id": "commonsense-011", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "A person who has not eaten all day is likely to feel", + "continuations": [ + " polished.", + " transparent.", + " hungry.", + " metallic." + ], + "tags": [ + "human_needs" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -21.305147171020508, + "mean_logprob": -7.101715723673503, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -21.010116577148438, + "mean_logprob": -10.505058288574219, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -5.818660736083984, + "mean_logprob": -1.9395535786946614, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -15.83749008178711, + "mean_logprob": -5.279163360595703, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 3.339609781901042, + "context_truncated": false, + "elapsed_seconds": 0.029702271000132896 + }, + { + "id": "commonsense-012", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "After washing wet clothes, a common next step is to", + "continuations": [ + " bury them in concrete.", + " mail them overseas.", + " paint them black.", + " dry them." + ], + "tags": [ + "everyday" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -20.121253967285156, + "mean_logprob": -3.3535423278808594, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -22.138946533203125, + "mean_logprob": -4.427789306640625, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -13.949477195739746, + "mean_logprob": -3.4873692989349365, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -4.874208450317383, + "mean_logprob": -1.6247361501057942, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 1.7288061777750652, + "context_truncated": false, + "elapsed_seconds": 0.0797503419998975 + }, + { + "id": "commonsense-013", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "If a candle flame is blown out, the room may become", + "continuations": [ + " darker.", + " heavier than a car.", + " filled with snow.", + " twice as wide." + ], + "tags": [ + "causality" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -6.059157371520996, + "mean_logprob": -2.019719123840332, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -27.804567337036133, + "mean_logprob": -4.6340945561726885, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -13.497331619262695, + "mean_logprob": -3.374332904815674, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -22.085975646972656, + "mean_logprob": -5.521493911743164, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 1.3546137809753418, + "context_truncated": false, + "elapsed_seconds": 0.035416651000559796 + }, + { + "id": "commonsense-014", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "To unlock a typical front door from outside, a resident may use", + "continuations": [ + " soap.", + " a key.", + " a pillow.", + " a forkful of rice." + ], + "tags": [ + "tool_use" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.688546180725098, + "mean_logprob": -7.844273090362549, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.20548152923584, + "mean_logprob": -2.4018271764119468, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -11.886775016784668, + "mean_logprob": -2.971693754196167, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -26.5023193359375, + "mean_logprob": -4.41705322265625, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.5698665777842202, + "context_truncated": false, + "elapsed_seconds": 0.03503433900004893 + }, + { + "id": "commonsense-015", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Fresh snow on a sidewalk can make the surface", + "continuations": [ + " spicy.", + " musical.", + " slippery.", + " edible." + ], + "tags": [ + "physical" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -16.030292510986328, + "mean_logprob": -5.343430836995442, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -18.539226531982422, + "mean_logprob": -9.269613265991211, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -3.111222743988037, + "mean_logprob": -0.7778056859970093, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -11.036014556884766, + "mean_logprob": -3.6786715189615884, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 2.900865832964579, + "context_truncated": false, + "elapsed_seconds": 0.03836592500010738 + }, + { + "id": "commonsense-016", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "If a smoke alarm sounds and smoke is visible, people should", + "continuations": [ + " finish a long nap.", + " hide the alarm.", + " open a novel.", + " leave the building safely." + ], + "tags": [ + "safety" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -31.506465911865234, + "mean_logprob": -5.251077651977539, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -15.628789901733398, + "mean_logprob": -3.9071974754333496, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -35.51291275024414, + "mean_logprob": -8.878228187561035, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -13.135072708129883, + "mean_logprob": -2.6270145416259765, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.2801829338073731, + "context_truncated": false, + "elapsed_seconds": 0.028125348999310518 + }, + { + "id": "commonsense-017", + "category": "commonsense", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "A ceramic mug filled with hot tea may be too hot to", + "continuations": [ + " hold comfortably.", + " remember a name.", + " become ceramic.", + " count backward." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -13.978292465209961, + "mean_logprob": -3.4945731163024902, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -26.312808990478516, + "mean_logprob": -6.578202247619629, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -22.872270584106445, + "mean_logprob": -5.718067646026611, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -23.461456298828125, + "mean_logprob": -7.820485432942708, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 2.223494529724121, + "context_truncated": false, + "elapsed_seconds": 0.03861710899946047 + }, + { + "id": "commonsense-018", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Tara heard thunder only a few seconds after seeing lightning, suggesting the storm was", + "continuations": [ + " beneath the ocean.", + " nearby.", + " made of glass.", + " over last year." + ], + "tags": [ + "causality" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -12.610671043395996, + "mean_logprob": -3.152667760848999, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -7.430370330810547, + "mean_logprob": -3.7151851654052734, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -14.351226806640625, + "mean_logprob": -3.5878067016601562, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -21.74671745300293, + "mean_logprob": -5.436679363250732, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.4351389408111572, + "context_truncated": false, + "elapsed_seconds": 0.02832317799948214 + }, + { + "id": "commonsense-019", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The freezer door was left open overnight. In the morning, the ice cream was probably", + "continuations": [ + " sharper.", + " newly packaged.", + " soft or melted.", + " louder." + ], + "tags": [ + "causality" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -15.146666526794434, + "mean_logprob": -3.7866666316986084, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -21.309471130371094, + "mean_logprob": -7.103157043457031, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -16.809839248657227, + "mean_logprob": -3.361967849731445, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -11.778133392333984, + "mean_logprob": -2.944533348083496, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.41743450164794904, + "context_truncated": false, + "elapsed_seconds": 0.035673541999130975 + }, + { + "id": "commonsense-020", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "Ravi arrived early for an interview but did not want to interrupt the meeting inside, so he", + "continuations": [ + " shouted through the door.", + " canceled every meeting.", + " moved the building.", + " waited quietly." + ], + "tags": [ + "social" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -18.4698486328125, + "mean_logprob": -2.6385498046875, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -21.761749267578125, + "mean_logprob": -3.626958211263021, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -17.902360916137695, + "mean_logprob": -4.475590229034424, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -10.329994201660156, + "mean_logprob": -2.065998840332031, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.5725509643554689, + "context_truncated": false, + "elapsed_seconds": 0.035349686000699876 + }, + { + "id": "commonsense-021", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "A recipe serves four people, but eight guests are expected. A sensible cook would", + "continuations": [ + " double the ingredients.", + " halve every plate.", + " freeze the oven.", + " omit all food." + ], + "tags": [ + "planning" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -11.00696086883545, + "mean_logprob": -2.7517402172088623, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -22.688758850097656, + "mean_logprob": -4.537751770019531, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -24.844209671020508, + "mean_logprob": -6.211052417755127, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -22.738433837890625, + "mean_logprob": -4.547686767578125, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.7860115528106686, + "context_truncated": false, + "elapsed_seconds": 0.027442759999757982 + }, + { + "id": "commonsense-022", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "Mina's laptop became very hot and its fan ran constantly, so she first checked whether", + "continuations": [ + " the keyboard knew her name.", + " its vents were blocked.", + " the screen was square.", + " noon had arrived." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -23.518367767333984, + "mean_logprob": -3.9197279612223306, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -16.88454246520996, + "mean_logprob": -2.8140904108683267, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -22.184528350830078, + "mean_logprob": -4.4369056701660154, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -25.38568115234375, + "mean_logprob": -5.07713623046875, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 1.105637550354004, + "context_truncated": false, + "elapsed_seconds": 0.027006951999283046 + }, + { + "id": "commonsense-023", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A friend says, \"I just lost my job.\" The most considerate immediate response is to", + "continuations": [ + " ask for a loan.", + " change the subject to sports.", + " express sympathy and listen.", + " congratulate them." + ], + "tags": [ + "social" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -12.615581512451172, + "mean_logprob": -2.5231163024902346, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -27.88450813293457, + "mean_logprob": -4.647418022155762, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -18.08245277404785, + "mean_logprob": -3.013742129007975, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -11.871034622192383, + "mean_logprob": -2.3742069244384765, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.14890937805175808, + "context_truncated": false, + "elapsed_seconds": 0.03359649000049103 + }, + { + "id": "commonsense-024", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The label says a medicine may cause drowsiness. After taking it, a person should avoid", + "continuations": [ + " drinking water.", + " reading the label.", + " sitting safely.", + " driving a car." + ], + "tags": [ + "safety" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -8.047860145568848, + "mean_logprob": -2.682620048522949, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -15.666956901550293, + "mean_logprob": -3.9167392253875732, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -23.298513412475586, + "mean_logprob": -7.766171137491862, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -14.242864608764648, + "mean_logprob": -3.560716152191162, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.8780961036682129, + "context_truncated": false, + "elapsed_seconds": 0.02827333200002613 + }, + { + "id": "commonsense-025", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "Water is dripping from the ceiling directly below an upstairs bathroom. The leak likely", + "continuations": [ + " originates near that bathroom.", + " comes from sunlight.", + " proves the roof is dry.", + " makes water weightless." + ], + "tags": [ + "diagnosis" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -21.697128295898438, + "mean_logprob": -3.0995897565569197, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -16.447704315185547, + "mean_logprob": -4.111926078796387, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -27.732826232910156, + "mean_logprob": -4.622137705485026, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -34.71245574951172, + "mean_logprob": -6.942491149902343, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.012336322239467, + "context_truncated": false, + "elapsed_seconds": 0.02621591200113471 + }, + { + "id": "commonsense-026", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The car's fuel gauge is nearly empty and the next town is far away. The prudent action is to", + "continuations": [ + " increase tire size.", + " refuel before leaving.", + " turn off the steering wheel.", + " ignore all signs." + ], + "tags": [ + "planning" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -23.950927734375, + "mean_logprob": -5.98773193359375, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -13.28018856048584, + "mean_logprob": -2.656037712097168, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -17.440576553344727, + "mean_logprob": -2.4915109361921037, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -15.294353485107422, + "mean_logprob": -3.8235883712768555, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.16452677590506415, + "context_truncated": false, + "elapsed_seconds": 0.03307416600000579 + }, + { + "id": "commonsense-027", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "A package marked \"fragile\" should generally be", + "continuations": [ + " shaken to test it.", + " left in rain.", + " handled carefully.", + " used as a chair." + ], + "tags": [ + "everyday" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -24.553762435913086, + "mean_logprob": -4.092293739318848, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -21.365097045898438, + "mean_logprob": -5.341274261474609, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -10.0580472946167, + "mean_logprob": -3.3526824315389, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -23.24994468688965, + "mean_logprob": -4.6499889373779295, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.7396113077799478, + "context_truncated": false, + "elapsed_seconds": 0.027997896000670153 + }, + { + "id": "commonsense-028", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "Kai cannot hear the television, but everyone else can. Before replacing it, Kai should check", + "continuations": [ + " whether the walls are blue.", + " how much the sofa weighs.", + " if tomorrow is Tuesday.", + " the volume and audio settings." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -26.885547637939453, + "mean_logprob": -4.480924606323242, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -25.82662582397461, + "mean_logprob": -3.6895179748535156, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -20.93070411682129, + "mean_logprob": -4.186140823364258, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -16.532907485961914, + "mean_logprob": -2.7554845809936523, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.9340333938598633, + "context_truncated": false, + "elapsed_seconds": 0.03174066499923356 + }, + { + "id": "commonsense-029", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A wet floor sign in a hallway warns pedestrians that they should", + "continuations": [ + " walk carefully.", + " run faster.", + " turn off every light.", + " remove their shoes immediately." + ], + "tags": [ + "safety" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -11.348014831542969, + "mean_logprob": -3.7826716105143228, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -11.512643814086914, + "mean_logprob": -3.837547938028971, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -27.827768325805664, + "mean_logprob": -5.565553665161133, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -9.475430488586426, + "mean_logprob": -1.8950860977172852, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.8875855127970376, + "context_truncated": false, + "elapsed_seconds": 0.02886738199958927 + }, + { + "id": "commonsense-030", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "Nico promised to return a borrowed book on Friday but needs it until Monday. The responsible choice is to", + "continuations": [ + " hide the book.", + " ask the owner for an extension.", + " claim Friday was canceled.", + " lend it to a stranger." + ], + "tags": [ + "social" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -13.196928024291992, + "mean_logprob": -3.299232006072998, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -20.57825469970703, + "mean_logprob": -2.9397506713867188, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -32.479103088378906, + "mean_logprob": -4.639871869768415, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -19.80704116821289, + "mean_logprob": -2.4758801460266113, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.4638705253601074, + "context_truncated": false, + "elapsed_seconds": 0.033459694001066964 + }, + { + "id": "commonsense-031", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "A refrigerator is running, but food inside is warm. One useful first check is whether", + "continuations": [ + " the food can read.", + " the kitchen is rectangular.", + " the door seals completely.", + " the shelves are named." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -22.93718147277832, + "mean_logprob": -4.587436294555664, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -19.133255004882812, + "mean_logprob": -3.8266510009765624, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -16.214479446411133, + "mean_logprob": -2.702413241068522, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -28.418724060058594, + "mean_logprob": -4.736454010009766, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.1242377599080404, + "context_truncated": false, + "elapsed_seconds": 0.02941785599978175 + }, + { + "id": "commonsense-032", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "A hiker sees dark clouds approaching and has several hours left on an exposed ridge. The safer choice is to", + "continuations": [ + " climb the tallest tree.", + " hold a metal pole.", + " wait for lightning.", + " descend toward shelter." + ], + "tags": [ + "safety" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -18.073514938354492, + "mean_logprob": -3.0122524897257485, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -25.604442596435547, + "mean_logprob": -5.120888519287109, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -17.805259704589844, + "mean_logprob": -3.561051940917969, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -18.156234741210938, + "mean_logprob": -4.539058685302734, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.5487994511922203, + "context_truncated": false, + "elapsed_seconds": 0.03707048300020688 + }, + { + "id": "commonsense-033", + "category": "commonsense", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "An egg dropped onto a thick cushion is less likely to break because the cushion", + "continuations": [ + " reduces the sudden impact.", + " makes the egg colder.", + " removes gravity.", + " turns shell into rubber." + ], + "tags": [ + "physical" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -19.05860710144043, + "mean_logprob": -3.811721420288086, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -20.780445098876953, + "mean_logprob": -3.463407516479492, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -20.995624542236328, + "mean_logprob": -6.998541514078776, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -35.17015838623047, + "mean_logprob": -7.034031677246094, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.34831390380859384, + "context_truncated": false, + "elapsed_seconds": 0.047453494000365026 + }, + { + "id": "commonsense-034", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A metal jar lid is stuck. Running warm water over the lid can help because the metal may", + "continuations": [ + " lose all mass.", + " expand slightly.", + " stop being metal.", + " become colder than ice." + ], + "tags": [ + "physical" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -26.746746063232422, + "mean_logprob": -6.6866865158081055, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.445197105407715, + "mean_logprob": -2.4817323684692383, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -22.945758819580078, + "mean_logprob": -5.7364397048950195, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -18.385082244873047, + "mean_logprob": -3.064180374145508, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.5824480056762695, + "context_truncated": false, + "elapsed_seconds": 0.027411420000134967 + }, + { + "id": "commonsense-035", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "Lena receives an email demanding immediate payment through gift cards and threatening arrest. The message is most likely", + "continuations": [ + " a routine tax receipt.", + " a weather alert.", + " a scam.", + " a library reminder." + ], + "tags": [ + "digital_safety" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -27.713577270507812, + "mean_logprob": -4.618929545084636, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -17.01344108581543, + "mean_logprob": -4.253360271453857, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -7.814584732055664, + "mean_logprob": -1.953646183013916, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -25.283443450927734, + "mean_logprob": -6.320860862731934, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 2.2997140884399414, + "context_truncated": false, + "elapsed_seconds": 0.03003643500051112 + }, + { + "id": "commonsense-036", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A sealed plastic bottle dents inward after hot air inside cools. The best explanation is that", + "continuations": [ + " the bottle gained extra air.", + " cooling erased the plastic.", + " gravity reversed briefly.", + " pressure inside decreased." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -26.828231811523438, + "mean_logprob": -4.471371968587239, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -29.029586791992188, + "mean_logprob": -4.838264465332031, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -37.96666717529297, + "mean_logprob": -9.491666793823242, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -23.304729461669922, + "mean_logprob": -5.8261823654174805, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.36689249674479196, + "context_truncated": false, + "elapsed_seconds": 0.027758326999901328 + }, + { + "id": "commonsense-037", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A colleague repeatedly misses deadlines because requirements change without notice. The most constructive first step is to", + "continuations": [ + " document changes and clarify priorities.", + " publicly insult the colleague.", + " silently redo all work forever.", + " delete the project." + ], + "tags": [ + "workplace" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -22.14117431640625, + "mean_logprob": -3.6901957194010415, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -24.30960464477539, + "mean_logprob": -3.4728006635393416, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -52.79871368408203, + "mean_logprob": -6.599839210510254, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -17.443693161010742, + "mean_logprob": -4.3609232902526855, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.21739505586169994, + "context_truncated": false, + "elapsed_seconds": 0.028819236000344972 + }, + { + "id": "commonsense-038", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A carbon monoxide alarm sounds, but nobody can smell smoke. Occupants should still", + "continuations": [ + " assume silence means safety.", + " leave and contact emergency services.", + " search with a flame.", + " unplug it and sleep." + ], + "tags": [ + "safety" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -34.20486831665039, + "mean_logprob": -6.840973663330078, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -21.1650333404541, + "mean_logprob": -3.52750555674235, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -27.941219329833984, + "mean_logprob": -4.656869888305664, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -27.197368621826172, + "mean_logprob": -3.885338374546596, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.3578328178042458, + "context_truncated": false, + "elapsed_seconds": 0.02605757099991024 + }, + { + "id": "commonsense-039", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A wooden drawer sticks only during humid weather. The most plausible reason is that the wood", + "continuations": [ + " becomes electrically charged.", + " forgets its shape.", + " swells after absorbing moisture.", + " turns into metal." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -19.181184768676758, + "mean_logprob": -3.196864128112793, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -23.117597579956055, + "mean_logprob": -4.623519515991211, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -30.014820098876953, + "mean_logprob": -4.2878314426967075, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -20.909446716308594, + "mean_logprob": -5.227361679077148, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.0909673145839145, + "context_truncated": false, + "elapsed_seconds": 0.03314090099956957 + }, + { + "id": "commonsense-040", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "Sam hears a rumor about a coworker from someone who admits they were not present. Before repeating it, Sam should", + "continuations": [ + " add dramatic details.", + " post it anonymously.", + " treat it as confirmed.", + " verify the information." + ], + "tags": [ + "social" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -29.395584106445312, + "mean_logprob": -7.348896026611328, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -21.053836822509766, + "mean_logprob": -3.5089728037516275, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -22.396867752075195, + "mean_logprob": -4.479373550415039, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -10.483184814453125, + "mean_logprob": -2.6207962036132812, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.8881766001383462, + "context_truncated": false, + "elapsed_seconds": 0.04679363399918657 + }, + { + "id": "commonsense-041", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "Two identical wet towels are hung up, one spread flat and one crumpled. The spread towel usually dries first because it has", + "continuations": [ + " more surface exposed to air.", + " less fabric.", + " no water initially.", + " a different mass forever." + ], + "tags": [ + "physical" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -14.11756706237793, + "mean_logprob": -2.352927843729655, + "total_sequence_tokens": 36 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -10.636102676391602, + "mean_logprob": -3.5453675587972007, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -17.9916934967041, + "mean_logprob": -4.497923374176025, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -29.345022201538086, + "mean_logprob": -5.869004440307617, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 1.1924397150675459, + "context_truncated": false, + "elapsed_seconds": 0.04707452600086981 + }, + { + "id": "commonsense-042", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A meeting invite lists 3 p.m. in another time zone. To avoid arriving at the wrong time, a participant should", + "continuations": [ + " ignore the time zone.", + " convert it to local time.", + " attend at every hour.", + " change the calendar date randomly." + ], + "tags": [ + "planning" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -17.2685604095459, + "mean_logprob": -3.4537120819091798, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -22.19762420654297, + "mean_logprob": -3.699604034423828, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -27.377838134765625, + "mean_logprob": -5.475567626953125, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -31.6478328704834, + "mean_logprob": -5.274638811747233, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.24589195251464835, + "context_truncated": false, + "elapsed_seconds": 0.04162225400068564 + }, + { + "id": "commonsense-043", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A pan of oil catches fire on a stove. The cook should turn off the heat and", + "continuations": [ + " pour water into the oil.", + " carry the burning pan outside.", + " cover the pan with a lid.", + " fan the flames." + ], + "tags": [ + "safety" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -14.20297622680664, + "mean_logprob": -2.3671627044677734, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -21.66763687133789, + "mean_logprob": -3.6112728118896484, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -10.082666397094727, + "mean_logprob": -1.4403809138706751, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -12.846022605895996, + "mean_logprob": -2.5692045211791994, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.9267817905970983, + "context_truncated": false, + "elapsed_seconds": 0.028579490000993246 + }, + { + "id": "commonsense-044", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "After a software update, a printer stops appearing on one computer but works from others. The issue is most likely with", + "continuations": [ + " all paper ever made.", + " the building's plumbing.", + " the printer's physical existence.", + " that computer's configuration." + ], + "tags": [ + "troubleshooting" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -44.12640380859375, + "mean_logprob": -8.82528076171875, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -24.53286361694336, + "mean_logprob": -3.06660795211792, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -23.706457138061523, + "mean_logprob": -3.386636734008789, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -15.543107986450195, + "mean_logprob": -2.590517997741699, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.4760899543762207, + "context_truncated": false, + "elapsed_seconds": 0.033685976999549894 + }, + { + "id": "commonsense-045", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A person feels dizzy after standing up very quickly. Sitting back down helps primarily by", + "continuations": [ + " reducing the immediate risk of falling.", + " making the floor warmer.", + " stopping all circulation.", + " increasing their height." + ], + "tags": [ + "safety" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -16.934162139892578, + "mean_logprob": -2.419166019984654, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -19.554096221923828, + "mean_logprob": -3.910819244384766, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -20.506786346435547, + "mean_logprob": -5.126696586608887, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -17.24651527404785, + "mean_logprob": -4.311628818511963, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.4916532244001117, + "context_truncated": false, + "elapsed_seconds": 0.03306646099917998 + }, + { + "id": "commonsense-046", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A sealed bag of chips puffs up during a drive to high altitude because outside air pressure", + "continuations": [ + " becomes much higher.", + " becomes lower.", + " changes the chips into gas.", + " disappears completely." + ], + "tags": [ + "physical" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -13.810972213745117, + "mean_logprob": -3.4527430534362793, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -9.870441436767578, + "mean_logprob": -3.290147145589193, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -30.123023986816406, + "mean_logprob": -5.020503997802734, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -16.583572387695312, + "mean_logprob": -4.145893096923828, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.16259590784708644, + "context_truncated": false, + "elapsed_seconds": 0.02561516199966718 + }, + { + "id": "commonsense-047", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A manager asks for feedback but becomes defensive whenever criticism is offered. Employees will likely become", + "continuations": [ + " more candid each time.", + " unable to remember work.", + " reluctant to speak honestly.", + " certain all criticism vanished." + ], + "tags": [ + "social" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -25.41448974609375, + "mean_logprob": -5.08289794921875, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -31.7397403717041, + "mean_logprob": -6.34794807434082, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -15.93793773651123, + "mean_logprob": -1.9922422170639038, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -42.678192138671875, + "mean_logprob": -7.1130320231119795, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 3.090655732154846, + "context_truncated": false, + "elapsed_seconds": 0.027545950999410707 + }, + { + "id": "commonsense-048", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A parked car has one window fogged on the inside while the others are clear. This suggests moisture is concentrated", + "continuations": [ + " inside the engine only.", + " on the road beneath.", + " in the sunlight.", + " near that window." + ], + "tags": [ + "diagnosis" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -18.106586456298828, + "mean_logprob": -3.6213172912597655, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -16.36924934387207, + "mean_logprob": -3.273849868774414, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -12.548701286315918, + "mean_logprob": -3.1371753215789795, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -12.029919624328613, + "mean_logprob": -3.0074799060821533, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.12969541549682617, + "context_truncated": false, + "elapsed_seconds": 0.034464940001271316 + }, + { + "id": "commonsense-049", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "A child apologizes only after being caught and immediately repeats the behavior. The apology alone is weak evidence of", + "continuations": [ + " genuine change.", + " having been caught.", + " the behavior occurring.", + " spoken language." + ], + "tags": [ + "social" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -12.416661262512207, + "mean_logprob": -4.138887087504069, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -17.955171585083008, + "mean_logprob": -4.488792896270752, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -14.714911460876465, + "mean_logprob": -3.678727865219116, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -21.599285125732422, + "mean_logprob": -7.199761708577474, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.4601592222849531, + "context_truncated": false, + "elapsed_seconds": 0.03128207199915778 + }, + { + "id": "commonsense-050", + "category": "commonsense", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A rechargeable battery lasts much less time in freezing weather but recovers indoors. The temporary loss is most plausibly due to", + "continuations": [ + " permanent deletion of electricity.", + " slower chemical reactions in the cold.", + " the device becoming heavier.", + " indoor clocks running faster." + ], + "tags": [ + "physical" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -31.399362564086914, + "mean_logprob": -5.233227094014485, + "total_sequence_tokens": 36 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -24.70549201965332, + "mean_logprob": -3.529356002807617, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -23.229263305664062, + "mean_logprob": -3.8715438842773438, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -34.779884338378906, + "mean_logprob": -5.796647389729817, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.34218788146972656, + "context_truncated": false, + "elapsed_seconds": 0.04696313400017971 + }, + { + "id": "world_knowledge-001", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The capital city of France is", + "continuations": [ + " Vienna.", + " Madrid.", + " Paris.", + " Lisbon." + ], + "tags": [ + "geography" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -11.509716033935547, + "mean_logprob": -3.836572011311849, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.032500743865967, + "mean_logprob": -2.344166914621989, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -3.3560290336608887, + "mean_logprob": -1.6780145168304443, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -9.288416862487793, + "mean_logprob": -2.3221042156219482, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.6440896987915039, + "context_truncated": false, + "elapsed_seconds": 0.03533909900033905 + }, + { + "id": "world_knowledge-002", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The planet known for its prominent rings is", + "continuations": [ + " Mercury.", + " Saturn.", + " Earth.", + " Mars." + ], + "tags": [ + "science" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -7.320735454559326, + "mean_logprob": -2.4402451515197754, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -8.424192428588867, + "mean_logprob": -4.212096214294434, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -6.013822555541992, + "mean_logprob": -3.006911277770996, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -8.402318954467773, + "mean_logprob": -4.201159477233887, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.5666661262512207, + "context_truncated": false, + "elapsed_seconds": 0.027122282999698655 + }, + { + "id": "world_knowledge-003", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Water freezes at zero degrees on the", + "continuations": [ + " Kelvin scale.", + " Fahrenheit scale.", + " Celsius scale.", + " Beaufort scale." + ], + "tags": [ + "science" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -8.779051780700684, + "mean_logprob": -1.7558103561401368, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -10.190202713012695, + "mean_logprob": -1.4557432447160994, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -6.731854438781738, + "mean_logprob": -1.3463708877563476, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -15.025651931762695, + "mean_logprob": -2.504275321960449, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.10937235695975178, + "context_truncated": false, + "elapsed_seconds": 0.02718069999900763 + }, + { + "id": "world_knowledge-004", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The author of the play Hamlet was", + "continuations": [ + " Charles Dickens.", + " Jane Austen.", + " Mark Twain.", + " William Shakespeare." + ], + "tags": [ + "literature" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -11.206514358520508, + "mean_logprob": -2.801628589630127, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -12.495871543884277, + "mean_logprob": -3.1239678859710693, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -15.107165336608887, + "mean_logprob": -3.7767913341522217, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -10.643210411071777, + "mean_logprob": -3.5477368036905923, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.3223392963409424, + "context_truncated": false, + "elapsed_seconds": 0.026172885000050883 + }, + { + "id": "world_knowledge-005", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The largest ocean on Earth is the", + "continuations": [ + " Pacific Ocean.", + " Indian Ocean.", + " Arctic Ocean.", + " Atlantic Ocean." + ], + "tags": [ + "geography" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -4.973231792449951, + "mean_logprob": -1.6577439308166504, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -5.313286304473877, + "mean_logprob": -1.7710954348246257, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -5.450287342071533, + "mean_logprob": -1.8167624473571777, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -5.077324867248535, + "mean_logprob": -1.6924416224161785, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.03469769159952807, + "context_truncated": false, + "elapsed_seconds": 0.03793094100001326 + }, + { + "id": "world_knowledge-006", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Plants primarily absorb water through their", + "continuations": [ + " flowers.", + " roots.", + " seeds.", + " fruit." + ], + "tags": [ + "biology" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.8627290725708, + "mean_logprob": -6.4313645362854, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -1.4958585500717163, + "mean_logprob": -0.7479292750358582, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -11.364479064941406, + "mean_logprob": -5.682239532470703, + "total_sequence_tokens": 9 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -13.464406967163086, + "mean_logprob": -6.732203483581543, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 4.934310257434845, + "context_truncated": false, + "elapsed_seconds": 0.031080873000973952 + }, + { + "id": "world_knowledge-007", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The chemical symbol for oxygen is", + "continuations": [ + " Ox.", + " G.", + " O.", + " Og." + ], + "tags": [ + "chemistry" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -10.892958641052246, + "mean_logprob": -3.630986213684082, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.250329971313477, + "mean_logprob": -6.625164985656738, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -4.206754684448242, + "mean_logprob": -2.103377342224121, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -11.048662185668945, + "mean_logprob": -3.682887395222982, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 1.527608871459961, + "context_truncated": false, + "elapsed_seconds": 0.027357783999832463 + }, + { + "id": "world_knowledge-008", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The Nile River flows through the continent of", + "continuations": [ + " South America.", + " Australia.", + " Europe.", + " Africa." + ], + "tags": [ + "geography" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -3.3431458473205566, + "mean_logprob": -1.1143819491068523, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -6.791843414306641, + "mean_logprob": -3.3959217071533203, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -5.591670036315918, + "mean_logprob": -2.795835018157959, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -1.2561030387878418, + "mean_logprob": -0.6280515193939209, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.4863304297129314, + "context_truncated": false, + "elapsed_seconds": 0.0381768369989004 + }, + { + "id": "world_knowledge-009", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "A triangle has", + "continuations": [ + " three sides.", + " four sides.", + " five sides.", + " six sides." + ], + "tags": [ + "mathematics" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -5.732003211975098, + "mean_logprob": -1.9106677373250325, + "total_sequence_tokens": 6 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -9.198739051818848, + "mean_logprob": -3.0662463506062827, + "total_sequence_tokens": 6 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -10.230048179626465, + "mean_logprob": -3.4100160598754883, + "total_sequence_tokens": 6 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -9.581594467163086, + "mean_logprob": -3.1938648223876953, + "total_sequence_tokens": 6 + } + ], + "best_vs_second_margin": 1.1555786132812502, + "context_truncated": false, + "elapsed_seconds": 0.030920102000891347 + }, + { + "id": "world_knowledge-010", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The organ that pumps blood through the human body is the", + "continuations": [ + " liver.", + " heart.", + " lung.", + " stomach." + ], + "tags": [ + "biology" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -4.063324928283691, + "mean_logprob": -2.0316624641418457, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -1.312350869178772, + "mean_logprob": -0.656175434589386, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -9.319375038146973, + "mean_logprob": -4.659687519073486, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -7.700719833374023, + "mean_logprob": -3.8503599166870117, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 1.3754870295524597, + "context_truncated": false, + "elapsed_seconds": 0.024139918001310434 + }, + { + "id": "world_knowledge-011", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "Japan is an island country in", + "continuations": [ + " South America.", + " western Africa.", + " East Asia.", + " northern Europe." + ], + "tags": [ + "geography" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -5.345488548278809, + "mean_logprob": -1.7818295160929363, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -8.188445091247559, + "mean_logprob": -2.7294816970825195, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -7.010624885559082, + "mean_logprob": -2.3368749618530273, + "total_sequence_tokens": 9 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -9.137768745422363, + "mean_logprob": -3.0459229151407876, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 0.5550454457600911, + "context_truncated": false, + "elapsed_seconds": 0.05065048700089392 + }, + { + "id": "world_knowledge-012", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The first month of the Gregorian calendar is", + "continuations": [ + " March.", + " June.", + " December.", + " January." + ], + "tags": [ + "calendar" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -7.40371036529541, + "mean_logprob": -3.701855182647705, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -7.013602256774902, + "mean_logprob": -3.506801128387451, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -7.550854682922363, + "mean_logprob": -3.7754273414611816, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -5.948097229003906, + "mean_logprob": -2.974048614501953, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.532752513885498, + "context_truncated": false, + "elapsed_seconds": 0.071897378000358 + }, + { + "id": "world_knowledge-013", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "Bees commonly produce", + "continuations": [ + " honey.", + " salt.", + " wool.", + " paper." + ], + "tags": [ + "biology" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -5.9243621826171875, + "mean_logprob": -2.9621810913085938, + "total_sequence_tokens": 6 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.732301712036133, + "mean_logprob": -5.866150856018066, + "total_sequence_tokens": 6 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -9.110575675964355, + "mean_logprob": -4.555287837982178, + "total_sequence_tokens": 6 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -11.831554412841797, + "mean_logprob": -5.915777206420898, + "total_sequence_tokens": 6 + } + ], + "best_vs_second_margin": 1.593106746673584, + "context_truncated": false, + "elapsed_seconds": 0.03012902299997222 + }, + { + "id": "world_knowledge-014", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "The Great Pyramid of Giza is located in", + "continuations": [ + " Peru.", + " Egypt.", + " India.", + " Greece." + ], + "tags": [ + "history" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -13.07826042175293, + "mean_logprob": -4.35942014058431, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -4.243206024169922, + "mean_logprob": -2.121603012084961, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.2263765335083, + "mean_logprob": -6.61318826675415, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -15.253130912780762, + "mean_logprob": -7.626565456390381, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 2.2378171284993487, + "context_truncated": false, + "elapsed_seconds": 0.03506938899954548 + }, + { + "id": "world_knowledge-015", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 0.0, + "item_elo": 800, + "context": "The gas humans need to breathe for cellular respiration is", + "continuations": [ + " helium.", + " neon.", + " oxygen.", + " argon." + ], + "tags": [ + "biology" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -12.004706382751465, + "mean_logprob": -3.001176595687866, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -22.805034637451172, + "mean_logprob": -7.601678212483724, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -6.715459823608398, + "mean_logprob": -3.357729911804199, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -11.974287986755371, + "mean_logprob": -3.991429328918457, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.356553316116333, + "context_truncated": false, + "elapsed_seconds": 0.027213841000047978 + }, + { + "id": "world_knowledge-016", + "category": "world_knowledge", + "difficulty": "easy", + "category_weight": 1.1, + "difficulty_weight": 1.0, + "point_weight": 1.1, + "earned_points": 1.1, + "item_elo": 800, + "context": "A standard chessboard contains", + "continuations": [ + " forty squares.", + " fifty squares.", + " one hundred squares.", + " sixty-four squares." + ], + "tags": [ + "culture" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -15.206071853637695, + "mean_logprob": -5.0686906178792315, + "total_sequence_tokens": 8 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -12.570915222167969, + "mean_logprob": -4.190305074055989, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -15.034111022949219, + "mean_logprob": -3.7585277557373047, + "total_sequence_tokens": 9 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -17.005380630493164, + "mean_logprob": -2.429340090070452, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 1.3291876656668529, + "context_truncated": false, + "elapsed_seconds": 0.027590434001467656 + }, + { + "id": "world_knowledge-017", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The treaty that formally ended World War I between Germany and the Allied powers was the Treaty of", + "continuations": [ + " Versailles.", + " Utrecht.", + " Tordesillas.", + " Ghent." + ], + "tags": [ + "history" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -0.7719295024871826, + "mean_logprob": -0.15438590049743653, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -11.48581600189209, + "mean_logprob": -2.8714540004730225, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -12.408638000488281, + "mean_logprob": -2.0681063334147134, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -8.774027824401855, + "mean_logprob": -2.924675941467285, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.9137204329172768, + "context_truncated": false, + "elapsed_seconds": 0.027041934999942896 + }, + { + "id": "world_knowledge-018", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The process by which plants convert light energy into chemical energy is", + "continuations": [ + " fermentation.", + " photosynthesis.", + " respiration only.", + " erosion." + ], + "tags": [ + "biology" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -15.712061882019043, + "mean_logprob": -5.237353960673015, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -12.029754638671875, + "mean_logprob": -3.0074386596679688, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -25.57769203186035, + "mean_logprob": -6.394423007965088, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -20.068880081176758, + "mean_logprob": -10.034440040588379, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 2.229915301005046, + "context_truncated": false, + "elapsed_seconds": 0.031013443000119878 + }, + { + "id": "world_knowledge-019", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The longest river entirely within Europe is the", + "continuations": [ + " Danube.", + " Rhine.", + " Volga.", + " Seine." + ], + "tags": [ + "geography" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -4.150540828704834, + "mean_logprob": -1.383513609568278, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -2.585611343383789, + "mean_logprob": -0.8618704477945963, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -5.604860782623291, + "mean_logprob": -1.868286927541097, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -6.248941898345947, + "mean_logprob": -2.0829806327819824, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.5216431617736818, + "context_truncated": false, + "elapsed_seconds": 0.03471019999960845 + }, + { + "id": "world_knowledge-020", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "In computing, RAM is primarily used for", + "continuations": [ + " permanent archival storage.", + " printing documents.", + " cooling the processor.", + " temporary working memory." + ], + "tags": [ + "computing" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -20.72980308532715, + "mean_logprob": -4.1459606170654295, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -17.150440216064453, + "mean_logprob": -5.716813405354817, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -16.436466217041016, + "mean_logprob": -4.109116554260254, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -15.068862915039062, + "mean_logprob": -3.7672157287597656, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.3419008255004883, + "context_truncated": false, + "elapsed_seconds": 0.03145416299958015 + }, + { + "id": "world_knowledge-021", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The scientist who formulated the three laws of motion was", + "continuations": [ + " Isaac Newton.", + " Louis Pasteur.", + " Gregor Mendel.", + " Niels Bohr." + ], + "tags": [ + "science_history" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -10.057504653930664, + "mean_logprob": -2.514376163482666, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -8.531913757324219, + "mean_logprob": -1.7063827514648438, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -10.613195419311523, + "mean_logprob": -1.7688659032185872, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -9.57681655883789, + "mean_logprob": -1.5961360931396484, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.11024665832519531, + "context_truncated": false, + "elapsed_seconds": 0.044894495000335155 + }, + { + "id": "world_knowledge-022", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The strait separating Spain from Morocco is the Strait of", + "continuations": [ + " Malacca.", + " Gibraltar.", + " Hormuz.", + " Bering." + ], + "tags": [ + "geography" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -5.7044291496276855, + "mean_logprob": -1.4261072874069214, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -1.5994062423706055, + "mean_logprob": -0.3198812484741211, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -3.2809271812438965, + "mean_logprob": -0.8202317953109741, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -15.879630088806152, + "mean_logprob": -5.293210029602051, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.500350546836853, + "context_truncated": false, + "elapsed_seconds": 0.03594739800064417 + }, + { + "id": "world_knowledge-023", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "A word with the opposite meaning of another word is called an", + "continuations": [ + " acronym.", + " homophone.", + " antonym.", + " anagram." + ], + "tags": [ + "language" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -7.133785247802734, + "mean_logprob": -1.7834463119506836, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -7.040304660797119, + "mean_logprob": -1.7600761651992798, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -0.9590001106262207, + "mean_logprob": -0.31966670354207355, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -2.9414892196655273, + "mean_logprob": -0.9804964065551758, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.6608297030131023, + "context_truncated": false, + "elapsed_seconds": 0.029652658000486554 + }, + { + "id": "world_knowledge-024", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The part of a cell that contains most of its genetic material is the", + "continuations": [ + " ribosome.", + " cell wall.", + " vacuole.", + " nucleus." + ], + "tags": [ + "biology" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -4.065311431884766, + "mean_logprob": -1.0163278579711914, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -11.278770446777344, + "mean_logprob": -3.7595901489257812, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -12.306262969970703, + "mean_logprob": -3.076565742492676, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -1.0450056791305542, + "mean_logprob": -0.5225028395652771, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.4938250184059143, + "context_truncated": false, + "elapsed_seconds": 0.02790486899903044 + }, + { + "id": "world_knowledge-025", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The Renaissance began as a major cultural movement in", + "continuations": [ + " Italy.", + " Iceland.", + " Canada.", + " Mongolia." + ], + "tags": [ + "history" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -3.302576780319214, + "mean_logprob": -1.651288390159607, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -12.38290786743164, + "mean_logprob": -3.09572696685791, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.717884063720703, + "mean_logprob": -6.858942031860352, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -18.62806510925293, + "mean_logprob": -6.2093550364176435, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 1.4444385766983032, + "context_truncated": false, + "elapsed_seconds": 0.028027561998897 + }, + { + "id": "world_knowledge-026", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The SI unit of electrical resistance is the", + "continuations": [ + " watt.", + " ohm.", + " tesla.", + " joule." + ], + "tags": [ + "physics" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -14.065362930297852, + "mean_logprob": -4.688454310099284, + "total_sequence_tokens": 11 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -1.9842658042907715, + "mean_logprob": -0.6614219347635905, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -14.778952598571777, + "mean_logprob": -3.6947381496429443, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -14.803125381469727, + "mean_logprob": -3.7007813453674316, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 3.033316214879354, + "context_truncated": false, + "elapsed_seconds": 0.028208798999912688 + }, + { + "id": "world_knowledge-027", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The novel Pride and Prejudice was written by", + "continuations": [ + " George Eliot.", + " Virginia Woolf.", + " Jane Austen.", + " Mary Shelley." + ], + "tags": [ + "literature" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -6.2252607345581055, + "mean_logprob": -1.5563151836395264, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -4.485273361206055, + "mean_logprob": -0.8970546722412109, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -2.3515846729278564, + "mean_logprob": -0.5878961682319641, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -8.91497802734375, + "mean_logprob": -1.78299560546875, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.3091585040092468, + "context_truncated": false, + "elapsed_seconds": 0.03339862100074242 + }, + { + "id": "world_knowledge-028", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The layer of Earth's atmosphere containing most weather is the", + "continuations": [ + " mesosphere.", + " thermosphere.", + " exosphere.", + " troposphere." + ], + "tags": [ + "earth_science" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -5.590150833129883, + "mean_logprob": -1.8633836110432942, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -6.984744548797607, + "mean_logprob": -2.3282481829325357, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -7.345436096191406, + "mean_logprob": -2.4484786987304688, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -2.125917911529541, + "mean_logprob": -0.5314794778823853, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.331904133160909, + "context_truncated": false, + "elapsed_seconds": 0.03057092500057479 + }, + { + "id": "world_knowledge-029", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The Magna Carta was sealed in", + "continuations": [ + " England.", + " Brazil.", + " China.", + " Ethiopia." + ], + "tags": [ + "history" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.145002365112305, + "mean_logprob": -5.572501182556152, + "total_sequence_tokens": 10 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -20.30511474609375, + "mean_logprob": -10.152557373046875, + "total_sequence_tokens": 10 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -15.851038932800293, + "mean_logprob": -7.9255194664001465, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -16.8714599609375, + "mean_logprob": -5.623819986979167, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.05131880442301462, + "context_truncated": false, + "elapsed_seconds": 0.04208292600014829 + }, + { + "id": "world_knowledge-030", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "In music, a group of three notes played together is commonly called a", + "continuations": [ + " scale.", + " chord.", + " tempo.", + " clef." + ], + "tags": [ + "music" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -6.692070960998535, + "mean_logprob": -3.3460354804992676, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -2.7004072666168213, + "mean_logprob": -1.3502036333084106, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -12.314258575439453, + "mean_logprob": -4.104752858479817, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -12.103459358215332, + "mean_logprob": -4.034486452738444, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.995831847190857, + "context_truncated": false, + "elapsed_seconds": 0.029503273000955232 + }, + { + "id": "world_knowledge-031", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 0.0, + "item_elo": 900, + "context": "The blood vessels that generally carry blood away from the heart are", + "continuations": [ + " capillaries only.", + " lymph nodes.", + " arteries.", + " alveoli." + ], + "tags": [ + "biology" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -17.37689208984375, + "mean_logprob": -3.47537841796875, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -9.757102012634277, + "mean_logprob": -3.2523673375447593, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -7.300205230712891, + "mean_logprob": -3.6501026153564453, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -13.829465866088867, + "mean_logprob": -3.457366466522217, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.20499912897745753, + "context_truncated": false, + "elapsed_seconds": 0.055547699999806355 + }, + { + "id": "world_knowledge-032", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The decimal number ten is written in binary as", + "continuations": [ + " 1001.", + " 1111.", + " 1100.", + " 1010." + ], + "tags": [ + "computing" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -7.5533013343811035, + "mean_logprob": -1.2588835557301838, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -14.360594749450684, + "mean_logprob": -2.3934324582417807, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -9.910626411437988, + "mean_logprob": -1.651771068572998, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -4.08213472366333, + "mean_logprob": -0.6803557872772217, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.5785277684529622, + "context_truncated": false, + "elapsed_seconds": 0.0371820000000298 + }, + { + "id": "world_knowledge-033", + "category": "world_knowledge", + "difficulty": "medium", + "category_weight": 1.1, + "difficulty_weight": 1.5, + "point_weight": 1.6500000000000001, + "earned_points": 1.6500000000000001, + "item_elo": 900, + "context": "The mountain range commonly used as part of the boundary between Europe and Asia is the", + "continuations": [ + " Ural Mountains.", + " Andes.", + " Alps.", + " Atlas Mountains." + ], + "tags": [ + "geography" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -5.6374831199646, + "mean_logprob": -1.40937077999115, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -6.830323219299316, + "mean_logprob": -2.2767744064331055, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -4.304528713226318, + "mean_logprob": -1.4348429044087727, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -7.810175895690918, + "mean_logprob": -1.9525439739227295, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.02547212441762281, + "context_truncated": false, + "elapsed_seconds": 0.026710719999755383 + }, + { + "id": "world_knowledge-034", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The principle that no two electrons in an atom can share all four quantum numbers is the", + "continuations": [ + " Doppler effect.", + " Pauli exclusion principle.", + " uncertainty of measurement.", + " Hubble law." + ], + "tags": [ + "physics" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -14.631590843200684, + "mean_logprob": -2.9263181686401367, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -6.522121429443359, + "mean_logprob": -1.3044242858886719, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -27.918291091918945, + "mean_logprob": -6.979572772979736, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -26.616729736328125, + "mean_logprob": -6.654182434082031, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.6218938827514648, + "context_truncated": false, + "elapsed_seconds": 0.027726984000764787 + }, + { + "id": "world_knowledge-035", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The 1648 agreements often associated with the modern state system are known as the Peace of", + "continuations": [ + " Augsburg.", + " Paris.", + " Westphalia.", + " Nijmegen." + ], + "tags": [ + "history" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -4.747247695922852, + "mean_logprob": -1.186811923980713, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -3.567901849746704, + "mean_logprob": -1.783950924873352, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -1.6582977771759033, + "mean_logprob": -0.41457444429397583, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -6.829130172729492, + "mean_logprob": -1.3658260345458983, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.7722374796867371, + "context_truncated": false, + "elapsed_seconds": 0.02695665899955202 + }, + { + "id": "world_knowledge-036", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "In linguistics, the smallest unit that carries meaning is a", + "continuations": [ + " phoneme in every case.", + " clause.", + " diphthong.", + " morpheme." + ], + "tags": [ + "linguistics" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -26.508514404296875, + "mean_logprob": -4.4180857340494795, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -6.442619800567627, + "mean_logprob": -3.2213099002838135, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -12.539463996887207, + "mean_logprob": -2.5078927993774416, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -4.539055347442627, + "mean_logprob": -1.1347638368606567, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.3731289625167848, + "context_truncated": false, + "elapsed_seconds": 0.02788937600053032 + }, + { + "id": "world_knowledge-037", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The enzyme that unwinds the DNA double helix during replication is", + "continuations": [ + " helicase.", + " amylase.", + " lipase.", + " pepsin." + ], + "tags": [ + "biology" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -7.697218418121338, + "mean_logprob": -1.9243046045303345, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -11.000181198120117, + "mean_logprob": -2.7500452995300293, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -12.166723251342773, + "mean_logprob": -4.055574417114258, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -11.399234771728516, + "mean_logprob": -3.7997449239095054, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.8257406949996948, + "context_truncated": false, + "elapsed_seconds": 0.025740410999787855 + }, + { + "id": "world_knowledge-038", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The economic concept describing the value of the best forgone alternative is", + "continuations": [ + " sunk cost.", + " opportunity cost.", + " marginal revenue.", + " price elasticity." + ], + "tags": [ + "economics" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -15.04764461517334, + "mean_logprob": -3.761911153793335, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.921466827392578, + "mean_logprob": -2.640488942464193, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -16.147680282592773, + "mean_logprob": -5.382560094197591, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -15.714448928833008, + "mean_logprob": -3.928612232208252, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.121422211329142, + "context_truncated": false, + "elapsed_seconds": 0.07778051700006472 + }, + { + "id": "world_knowledge-039", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The literary device in which an absent or imaginary person is directly addressed is", + "continuations": [ + " metonymy.", + " synecdoche.", + " apostrophe.", + " enjambment." + ], + "tags": [ + "literature" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -9.08511734008789, + "mean_logprob": -2.2712793350219727, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -10.178845405578613, + "mean_logprob": -2.0357690811157227, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -12.251620292663574, + "mean_logprob": -2.450324058532715, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -13.08072280883789, + "mean_logprob": -2.616144561767578, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.2355102539062499, + "context_truncated": false, + "elapsed_seconds": 0.027328506999765523 + }, + { + "id": "world_knowledge-040", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The ocean current that carries warm water across the North Atlantic is commonly called the", + "continuations": [ + " Humboldt Current.", + " Benguela Current.", + " Oyashio Current.", + " Gulf Stream." + ], + "tags": [ + "earth_science" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -7.287515640258789, + "mean_logprob": -1.2145859400431316, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -9.331193923950195, + "mean_logprob": -1.5551989873250325, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -22.413164138793945, + "mean_logprob": -3.7355273564656577, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -0.9502784609794617, + "mean_logprob": -0.31675948699315387, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.8978264530499778, + "context_truncated": false, + "elapsed_seconds": 0.027548702000785852 + }, + { + "id": "world_knowledge-041", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The alloy traditionally made primarily from copper and tin is", + "continuations": [ + " bronze.", + " brass.", + " steel.", + " pewter." + ], + "tags": [ + "materials" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -10.973139762878418, + "mean_logprob": -3.657713254292806, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -8.976274490356445, + "mean_logprob": -2.992091496785482, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -10.525206565856934, + "mean_logprob": -5.262603282928467, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -15.067255020141602, + "mean_logprob": -3.7668137550354004, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.6656217575073242, + "context_truncated": false, + "elapsed_seconds": 0.02840788299909036 + }, + { + "id": "world_knowledge-042", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The mathematical constant that is the base of natural logarithms is", + "continuations": [ + " pi.", + " e.", + " phi.", + " tau." + ], + "tags": [ + "mathematics" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -6.910679340362549, + "mean_logprob": -3.4553396701812744, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -3.696976900100708, + "mean_logprob": -1.848488450050354, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -12.537050247192383, + "mean_logprob": -6.268525123596191, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -18.091705322265625, + "mean_logprob": -9.045852661132812, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 1.6068512201309204, + "context_truncated": false, + "elapsed_seconds": 0.02433855900017079 + }, + { + "id": "world_knowledge-043", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "In databases, a key that uniquely identifies each row in a table is a", + "continuations": [ + " foreign schema.", + " null index.", + " primary key.", + " stored procedure." + ], + "tags": [ + "computing" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -24.418195724487305, + "mean_logprob": -8.139398574829102, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -12.958831787109375, + "mean_logprob": -4.319610595703125, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -3.394084930419922, + "mean_logprob": -1.1313616434733074, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -9.721197128295898, + "mean_logprob": -3.2403990427652993, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 2.109037399291992, + "context_truncated": false, + "elapsed_seconds": 0.02499480299957213 + }, + { + "id": "world_knowledge-044", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The branch of philosophy concerned with the nature and limits of knowledge is", + "continuations": [ + " aesthetics.", + " ethics.", + " ontology alone.", + " epistemology." + ], + "tags": [ + "philosophy" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -12.635092735290527, + "mean_logprob": -4.211697578430176, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -12.695484161376953, + "mean_logprob": -6.347742080688477, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -20.721094131469727, + "mean_logprob": -5.180273532867432, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -7.288669586181641, + "mean_logprob": -1.4577339172363282, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 2.753963661193848, + "context_truncated": false, + "elapsed_seconds": 0.0514239969998016 + }, + { + "id": "world_knowledge-045", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The element with atomic number 74 is", + "continuations": [ + " tungsten.", + " titanium.", + " tantalum.", + " tellurium." + ], + "tags": [ + "chemistry" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -9.828885078430176, + "mean_logprob": -1.9657770156860352, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -10.466218948364258, + "mean_logprob": -3.4887396494547525, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -10.116809844970703, + "mean_logprob": -2.0233619689941404, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -11.358242988586426, + "mean_logprob": -2.8395607471466064, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.0575849533081052, + "context_truncated": false, + "elapsed_seconds": 0.027971211999101797 + }, + { + "id": "world_knowledge-046", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The ancient writing system used for the Mycenaean Greek language is", + "continuations": [ + " cuneiform.", + " Linear B.", + " ogham.", + " hieratic." + ], + "tags": [ + "history" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -8.766053199768066, + "mean_logprob": -2.1915132999420166, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.610755920410156, + "mean_logprob": -2.5369186401367188, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -20.13168716430664, + "mean_logprob": -5.03292179107666, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -12.041725158691406, + "mean_logprob": -3.0104312896728516, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.34540534019470215, + "context_truncated": false, + "elapsed_seconds": 0.026163692000409355 + }, + { + "id": "world_knowledge-047", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "A star's spectral class in the sequence OBAFGKM is primarily related to its", + "continuations": [ + " distance from Earth.", + " number of planets.", + " surface temperature.", + " age alone." + ], + "tags": [ + "astronomy" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -8.229168891906738, + "mean_logprob": -2.0572922229766846, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -18.440418243408203, + "mean_logprob": -4.610104560852051, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -5.304070949554443, + "mean_logprob": -1.7680236498514812, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -14.59379768371582, + "mean_logprob": -4.864599227905273, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.2892685731252034, + "context_truncated": false, + "elapsed_seconds": 0.02666007100015122 + }, + { + "id": "world_knowledge-048", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The grammatical case typically marking direct address is the", + "continuations": [ + " accusative.", + " genitive.", + " instrumental.", + " vocative." + ], + "tags": [ + "linguistics" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -15.903148651123047, + "mean_logprob": -3.9757871627807617, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -15.181949615478516, + "mean_logprob": -5.060649871826172, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -17.94576072692871, + "mean_logprob": -5.98192024230957, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -15.720349311828613, + "mean_logprob": -5.240116437276204, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 1.0848627090454102, + "context_truncated": false, + "elapsed_seconds": 0.03568337000069732 + }, + { + "id": "world_knowledge-049", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 2.475, + "item_elo": 1050, + "context": "The boundary around a black hole beyond which light cannot escape is the", + "continuations": [ + " event horizon.", + " heliopause.", + " Roche limit.", + " photosphere." + ], + "tags": [ + "physics" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -1.2948393821716309, + "mean_logprob": -0.43161312739054364, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -7.611450672149658, + "mean_logprob": -1.5222901344299316, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -10.070302963256836, + "mean_logprob": -2.517575740814209, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -11.192702293395996, + "mean_logprob": -3.730900764465332, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.0906770070393879, + "context_truncated": false, + "elapsed_seconds": 0.030300678001367487 + }, + { + "id": "world_knowledge-050", + "category": "world_knowledge", + "difficulty": "hard", + "category_weight": 1.1, + "difficulty_weight": 2.25, + "point_weight": 2.475, + "earned_points": 0.0, + "item_elo": 1050, + "context": "In rhetoric, an argument that attacks the person rather than the claim is called", + "continuations": [ + " reductio ad absurdum.", + " ad hominem.", + " modus ponens.", + " false equivalence only." + ], + "tags": [ + "rhetoric" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -8.812560081481934, + "mean_logprob": -1.1015700101852417, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -9.008734703063965, + "mean_logprob": -1.801746940612793, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -17.020626068115234, + "mean_logprob": -2.836771011352539, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -30.69535255432129, + "mean_logprob": -7.673838138580322, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.7001769304275514, + "context_truncated": false, + "elapsed_seconds": 0.035687636000147904 + }, + { + "id": "context_tracking-001", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "On the shelf, the red box contains tea and the blue box contains rice. The tea is in", + "continuations": [ + " the red box.", + " the blue box.", + " both boxes.", + " neither box." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -6.783926486968994, + "mean_logprob": -1.6959816217422485, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -7.442936897277832, + "mean_logprob": -1.860734224319458, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -11.799026489257812, + "mean_logprob": -3.933008829752604, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -19.635343551635742, + "mean_logprob": -6.545114517211914, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.16475260257720947, + "context_truncated": false, + "elapsed_seconds": 0.028946387999894796 + }, + { + "id": "context_tracking-002", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Ivo chose the violin, while Mara chose the flute. Mara's instrument is", + "continuations": [ + " the violin.", + " unspecified.", + " the drum.", + " the flute." + ], + "tags": [ + "binding" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -7.989151477813721, + "mean_logprob": -1.9972878694534302, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.466575622558594, + "mean_logprob": -6.733287811279297, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -11.59686279296875, + "mean_logprob": -3.8656209309895835, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -5.811376094818115, + "mean_logprob": -1.4528440237045288, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.5444438457489014, + "context_truncated": false, + "elapsed_seconds": 0.02690631699988444 + }, + { + "id": "context_tracking-003", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "The morning bus stops at Oak Street. The evening bus stops at Pine Street. The evening stop is", + "continuations": [ + " Oak Street.", + " Maple Street.", + " Pine Street.", + " Cedar Street." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -3.7684435844421387, + "mean_logprob": -1.256147861480713, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -8.26771354675293, + "mean_logprob": -2.0669283866882324, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -4.313518524169922, + "mean_logprob": -1.0783796310424805, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -9.800602912902832, + "mean_logprob": -1.9601205825805663, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 0.17776823043823242, + "context_truncated": false, + "elapsed_seconds": 0.029487604000678402 + }, + { + "id": "context_tracking-004", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Nia's badge number is 27, and Sol's badge number is 41. Badge 41 belongs to", + "continuations": [ + " Nia.", + " both people.", + " nobody.", + " Sol." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -8.971212387084961, + "mean_logprob": -2.9904041290283203, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -16.68063735961914, + "mean_logprob": -5.56021245320638, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -14.08039379119873, + "mean_logprob": -4.693464597066243, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -7.419593334197998, + "mean_logprob": -3.709796667098999, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.7193925380706787, + "context_truncated": false, + "elapsed_seconds": 0.0441296480003075 + }, + { + "id": "context_tracking-005", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The north garden grows mint. The south garden grows basil. Mint grows in", + "continuations": [ + " the north garden.", + " the south garden.", + " both gardens.", + " the greenhouse." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -4.372170448303223, + "mean_logprob": -1.0930426120758057, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -4.286496639251709, + "mean_logprob": -1.0716241598129272, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -6.398435592651367, + "mean_logprob": -2.1328118642171225, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -5.518257141113281, + "mean_logprob": -1.8394190470377605, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.021418452262878418, + "context_truncated": false, + "elapsed_seconds": 0.07577670899991062 + }, + { + "id": "context_tracking-006", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Room A is painted yellow; Room B is painted green. The green room is", + "continuations": [ + " Room A.", + " Room B.", + " both rooms.", + " Room C." + ], + "tags": [ + "binding" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -19.31879425048828, + "mean_logprob": -6.439598083496094, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -20.013511657714844, + "mean_logprob": -6.671170552571614, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -15.426803588867188, + "mean_logprob": -5.1422678629557295, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -14.464043617248535, + "mean_logprob": -4.821347872416179, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.3209199905395508, + "context_truncated": false, + "elapsed_seconds": 0.031925275001412956 + }, + { + "id": "context_tracking-007", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "At noon, the cafe serves soup. At night, it serves noodles. The night menu features", + "continuations": [ + " soup.", + " salad.", + " noodles.", + " bread only." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -15.040764808654785, + "mean_logprob": -7.520382404327393, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -18.535228729248047, + "mean_logprob": -6.178409576416016, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -14.340347290039062, + "mean_logprob": -3.5850868225097656, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -27.44425392150879, + "mean_logprob": -9.14808464050293, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 2.59332275390625, + "context_truncated": false, + "elapsed_seconds": 0.08141225300096266 + }, + { + "id": "context_tracking-008", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Paz owns the bicycle, and Ren owns the scooter. The scooter belongs to", + "continuations": [ + " Paz.", + " both Paz and Ren.", + " neither person.", + " Ren." + ], + "tags": [ + "binding" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -8.017608642578125, + "mean_logprob": -2.6725362141927085, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -14.909689903259277, + "mean_logprob": -2.4849483172098794, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -17.485139846801758, + "mean_logprob": -5.828379948933919, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -3.9368906021118164, + "mean_logprob": -1.9684453010559082, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.5165030161539712, + "context_truncated": false, + "elapsed_seconds": 0.035989186999358935 + }, + { + "id": "context_tracking-009", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "The silver key opens the attic. The copper key opens the cellar. The attic key is", + "continuations": [ + " silver.", + " copper.", + " wooden.", + " black." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -7.160383701324463, + "mean_logprob": -3.5801918506622314, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -9.434481620788574, + "mean_logprob": -4.717240810394287, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -9.718422889709473, + "mean_logprob": -4.859211444854736, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -8.321786880493164, + "mean_logprob": -4.160893440246582, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.5807015895843506, + "context_truncated": false, + "elapsed_seconds": 0.03191114499895775 + }, + { + "id": "context_tracking-010", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Toma ordered pears, while Eda ordered plums. Eda ordered", + "continuations": [ + " pears.", + " plums.", + " peaches.", + " both fruits." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -3.681889057159424, + "mean_logprob": -1.2272963523864746, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -3.5526955127716064, + "mean_logprob": -1.1842318375905354, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -3.6570119857788086, + "mean_logprob": -1.2190039952596028, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -8.857694625854492, + "mean_logprob": -2.9525648752848306, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.03477215766906738, + "context_truncated": false, + "elapsed_seconds": 0.029180496001572465 + }, + { + "id": "context_tracking-011", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The first train goes to Linz. The second goes to Graz. The second destination is", + "continuations": [ + " Linz.", + " Vienna.", + " Graz.", + " Salzburg." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -3.7210230827331543, + "mean_logprob": -1.240341027577718, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -6.172287464141846, + "mean_logprob": -2.0574291547139487, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -5.512436866760254, + "mean_logprob": -1.8374789555867512, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -10.283622741699219, + "mean_logprob": -2.5709056854248047, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.5971379280090332, + "context_truncated": false, + "elapsed_seconds": 0.03351690399904328 + }, + { + "id": "context_tracking-012", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "File K is dated March, and File M is dated June. The June file is", + "continuations": [ + " File K.", + " both files.", + " undated.", + " File M." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -16.309968948364258, + "mean_logprob": -5.436656316121419, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -18.492210388183594, + "mean_logprob": -6.164070129394531, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -9.313602447509766, + "mean_logprob": -3.104534149169922, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -19.71459197998047, + "mean_logprob": -6.571530659993489, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 2.332122166951497, + "context_truncated": false, + "elapsed_seconds": 0.030803358000412118 + }, + { + "id": "context_tracking-013", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Luma wears a square pin. Oren wears a round pin. Luma's pin is", + "continuations": [ + " square.", + " round.", + " triangular.", + " missing." + ], + "tags": [ + "binding" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -6.6852593421936035, + "mean_logprob": -3.3426296710968018, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -8.037212371826172, + "mean_logprob": -4.018606185913086, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -8.829096794128418, + "mean_logprob": -4.414548397064209, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -7.15470027923584, + "mean_logprob": -3.57735013961792, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.23472046852111816, + "context_truncated": false, + "elapsed_seconds": 0.026890174000072875 + }, + { + "id": "context_tracking-014", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The code for apple is 6. The code for orange is 9. Code 9 means", + "continuations": [ + " apple.", + " orange.", + " pear.", + " fruit in general." + ], + "tags": [ + "binding" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.063654899597168, + "mean_logprob": -6.031827449798584, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.191564559936523, + "mean_logprob": -6.595782279968262, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -16.624719619750977, + "mean_logprob": -5.541573206583659, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -25.945880889892578, + "mean_logprob": -6.4864702224731445, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.49025424321492483, + "context_truncated": false, + "elapsed_seconds": 0.027514989000337664 + }, + { + "id": "context_tracking-015", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 1.2, + "item_elo": 850, + "context": "Kira arrived before Bo, and Bo arrived before Luis. The last to arrive was", + "continuations": [ + " Kira.", + " Bo.", + " Luis.", + " nobody." + ], + "tags": [ + "order" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -6.463204383850098, + "mean_logprob": -2.1544014612833657, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -6.712535858154297, + "mean_logprob": -3.3562679290771484, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -3.134509563446045, + "mean_logprob": -1.0448365211486816, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -11.898235321044922, + "mean_logprob": -3.966078440348307, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.109564940134684, + "context_truncated": false, + "elapsed_seconds": 0.027884559000085574 + }, + { + "id": "context_tracking-016", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "The small parcel weighs two kilos. The large parcel weighs five. The heavier parcel is", + "continuations": [ + " the small parcel.", + " both equally.", + " unknown.", + " the large parcel." + ], + "tags": [ + "comparison" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -12.038433074951172, + "mean_logprob": -2.4076866149902343, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -22.769210815429688, + "mean_logprob": -7.5897369384765625, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -11.203941345214844, + "mean_logprob": -5.601970672607422, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -13.794227600097656, + "mean_logprob": -2.7588455200195314, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.35115890502929714, + "context_truncated": false, + "elapsed_seconds": 0.02827097000044887 + }, + { + "id": "context_tracking-017", + "category": "context_tracking", + "difficulty": "easy", + "category_weight": 1.2, + "difficulty_weight": 1.0, + "point_weight": 1.2, + "earned_points": 0.0, + "item_elo": 850, + "context": "Tuesday's password is comet. Wednesday's password is lantern. On Tuesday, the password is", + "continuations": [ + " comet.", + " lantern.", + " Tuesday.", + " Wednesday." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -7.41854190826416, + "mean_logprob": -2.47284730275472, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -4.859280586242676, + "mean_logprob": -1.214820146560669, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -8.315832138061523, + "mean_logprob": -4.157916069030762, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -8.089353561401367, + "mean_logprob": -4.044676780700684, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 1.2580271561940513, + "context_truncated": false, + "elapsed_seconds": 0.03253696600040712 + }, + { + "id": "context_tracking-018", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Three couriers carry different folders: Aya has violet, Ben has white, and Cy has black. The white folder is carried by", + "continuations": [ + " Aya.", + " Ben.", + " Cy.", + " no courier." + ], + "tags": [ + "multi_binding" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -4.21363639831543, + "mean_logprob": -1.4045454661051433, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -6.199267864227295, + "mean_logprob": -3.0996339321136475, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -7.0412163734436035, + "mean_logprob": -3.5206081867218018, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -14.700834274291992, + "mean_logprob": -3.675208568572998, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 1.6950884660085042, + "context_truncated": false, + "elapsed_seconds": 0.034491018999688094 + }, + { + "id": "context_tracking-019", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "The gallery opens after the cafe but before the theater. The theater opens after the library. Among the gallery, cafe, and theater, the earliest is", + "continuations": [ + " the gallery.", + " the theater.", + " the cafe.", + " impossible to know." + ], + "tags": [ + "order" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -7.34194278717041, + "mean_logprob": -2.4473142623901367, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -6.660017967224121, + "mean_logprob": -2.220005989074707, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -5.666388034820557, + "mean_logprob": -1.4165970087051392, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -20.071624755859375, + "mean_logprob": -5.017906188964844, + "total_sequence_tokens": 36 + } + ], + "best_vs_second_margin": 0.8034089803695679, + "context_truncated": false, + "elapsed_seconds": 0.04306272899884789 + }, + { + "id": "context_tracking-020", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "A token moves from Box A to Box C, then from Box C to Box B. Its final location is", + "continuations": [ + " Box A.", + " Box C.", + " every box.", + " Box B." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -2.557363986968994, + "mean_logprob": -0.852454662322998, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -5.336918354034424, + "mean_logprob": -1.7789727846781414, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -21.708280563354492, + "mean_logprob": -7.236093521118164, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -2.3088560104370117, + "mean_logprob": -0.7696186701456705, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.08283599217732751, + "context_truncated": false, + "elapsed_seconds": 0.025915561000147136 + }, + { + "id": "context_tracking-021", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "Rin lends her map to Dev. Dev then gives the map back to Rin. At the end, the map is with", + "continuations": [ + " Rin.", + " Dev.", + " both of them.", + " nobody." + ], + "tags": [ + "state_update" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -2.3433592319488525, + "mean_logprob": -0.7811197439829508, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -4.1370415687561035, + "mean_logprob": -2.0685207843780518, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -10.318612098693848, + "mean_logprob": -2.579653024673462, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -17.195390701293945, + "mean_logprob": -5.731796900431315, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 1.287401040395101, + "context_truncated": false, + "elapsed_seconds": 0.02816087800056266 + }, + { + "id": "context_tracking-022", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The amber team scored below the teal team but above the gray team. The highest-scoring of these teams was", + "continuations": [ + " amber.", + " teal.", + " gray.", + " tied." + ], + "tags": [ + "order" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -11.838001251220703, + "mean_logprob": -3.946000417073568, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -13.28606128692627, + "mean_logprob": -4.42868709564209, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -12.694733619689941, + "mean_logprob": -6.347366809844971, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -13.777946472167969, + "mean_logprob": -6.888973236083984, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.482686678568522, + "context_truncated": false, + "elapsed_seconds": 0.029178336000768468 + }, + { + "id": "context_tracking-023", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "Ava's shift is Monday, Bram's is Thursday, and Cato's is Tuesday. The person working between Ava and Bram in the week is", + "continuations": [ + " Ava.", + " Bram.", + " Cato.", + " nobody." + ], + "tags": [ + "calendar" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -6.949986457824707, + "mean_logprob": -2.316662152608236, + "total_sequence_tokens": 39 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -6.109921932220459, + "mean_logprob": -2.0366406440734863, + "total_sequence_tokens": 39 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -4.568081378936768, + "mean_logprob": -1.5226937929789226, + "total_sequence_tokens": 39 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -12.695931434631348, + "mean_logprob": -4.231977144877116, + "total_sequence_tokens": 39 + } + ], + "best_vs_second_margin": 0.5139468510945637, + "context_truncated": false, + "elapsed_seconds": 0.046897251000700635 + }, + { + "id": "context_tracking-024", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "The note began under the lamp. Jo moved it to the desk, and Uma moved it from the desk into a drawer. The note ended", + "continuations": [ + " under the lamp.", + " on the desk.", + " with Jo.", + " in the drawer." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -6.655835151672363, + "mean_logprob": -1.6639587879180908, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -10.382094383239746, + "mean_logprob": -2.5955235958099365, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -11.422002792358398, + "mean_logprob": -3.8073342641194663, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -7.003646373748779, + "mean_logprob": -1.400729274749756, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.26322951316833487, + "context_truncated": false, + "elapsed_seconds": 0.047904680999636184 + }, + { + "id": "context_tracking-025", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "The labels are reassigned: north now means left, south means right, east means up, and west means down. Under this rule, north means", + "continuations": [ + " left.", + " right.", + " up.", + " down." + ], + "tags": [ + "remapping" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -4.648298263549805, + "mean_logprob": -2.3241491317749023, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -5.928032398223877, + "mean_logprob": -2.9640161991119385, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -5.894533634185791, + "mean_logprob": -2.9472668170928955, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -5.785933494567871, + "mean_logprob": -2.8929667472839355, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.5688176155090332, + "context_truncated": false, + "elapsed_seconds": 0.034175955999671714 + }, + { + "id": "context_tracking-026", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The bronze cup is taller than the glass cup. The glass cup is taller than the clay cup. The shortest is", + "continuations": [ + " bronze.", + " clay.", + " glass.", + " not specified." + ], + "tags": [ + "transitive" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -9.209604263305664, + "mean_logprob": -3.0698680877685547, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -10.724428176879883, + "mean_logprob": -5.362214088439941, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -8.372103691101074, + "mean_logprob": -4.186051845550537, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -10.977124214172363, + "mean_logprob": -3.659041404724121, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.5891733169555664, + "context_truncated": false, + "elapsed_seconds": 0.02894302099957713 + }, + { + "id": "context_tracking-027", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "Nell assigns fox to 14, owl to 22, and hare to 31. The animal assigned 31 is", + "continuations": [ + " fox.", + " owl.", + " hare.", + " none." + ], + "tags": [ + "binding" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -10.879972457885742, + "mean_logprob": -5.439986228942871, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -10.412038803100586, + "mean_logprob": -3.470679601033529, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -11.36612319946289, + "mean_logprob": -3.788707733154297, + "total_sequence_tokens": 35 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -14.26858139038086, + "mean_logprob": -7.13429069519043, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.3180281321207681, + "context_truncated": false, + "elapsed_seconds": 0.03798832400025276 + }, + { + "id": "context_tracking-028", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "A meeting moves from Room 3 to Room 8, then is moved back to its original room. It will occur in", + "continuations": [ + " Room 8.", + " both rooms.", + " no room.", + " Room 3." + ], + "tags": [ + "state_update" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -4.633544921875, + "mean_logprob": -1.15838623046875, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -10.276191711425781, + "mean_logprob": -3.4253972371419272, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -16.473928451538086, + "mean_logprob": -5.491309483846028, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -6.828451156616211, + "mean_logprob": -1.7071127891540527, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.5487265586853027, + "context_truncated": false, + "elapsed_seconds": 0.03693265699985204 + }, + { + "id": "context_tracking-029", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "In a made-up calendar, Dax follows Rul, and Vem follows Dax. The day immediately after Rul is", + "continuations": [ + " Dax.", + " Vem.", + " Rul.", + " unknown." + ], + "tags": [ + "local_rules" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -7.311236381530762, + "mean_logprob": -2.4370787938435874, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -6.153491973876953, + "mean_logprob": -2.051163991292318, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -9.810539245605469, + "mean_logprob": -3.2701797485351562, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -7.88333797454834, + "mean_logprob": -3.94166898727417, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.38591480255126953, + "context_truncated": false, + "elapsed_seconds": 0.03235356599907391 + }, + { + "id": "context_tracking-030", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "Faye is older than Gus but younger than Hana. The oldest of the three is", + "continuations": [ + " Faye.", + " Hana.", + " Gus.", + " tied." + ], + "tags": [ + "transitive" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -6.905505180358887, + "mean_logprob": -1.7263762950897217, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -3.8858604431152344, + "mean_logprob": -1.2952868143717449, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -3.9375972747802734, + "mean_logprob": -1.3125324249267578, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -12.933774948120117, + "mean_logprob": -6.466887474060059, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.017245610555012947, + "context_truncated": false, + "elapsed_seconds": 0.04053015899989987 + }, + { + "id": "context_tracking-031", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "The red route reaches Delta before Echo. The blue route reaches Echo before Delta. On the blue route, the first of those stops is", + "continuations": [ + " Delta.", + " neither stop.", + " Echo.", + " both together." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -6.452212810516357, + "mean_logprob": -3.2261064052581787, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -18.530963897705078, + "mean_logprob": -6.176987965901692, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -4.69672155380249, + "mean_logprob": -1.5655738512674968, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -21.00954818725586, + "mean_logprob": -7.003182729085286, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 1.660532553990682, + "context_truncated": false, + "elapsed_seconds": 0.03073819699966407 + }, + { + "id": "context_tracking-032", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "Iris gives the ring to Juno, Juno gives it to Kato, and Kato keeps it. The ring is finally with", + "continuations": [ + " Iris.", + " Juno.", + " everyone.", + " Kato." + ], + "tags": [ + "state_update" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -11.189102172851562, + "mean_logprob": -3.729700724283854, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -8.019768714904785, + "mean_logprob": -2.673256238301595, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -10.497590065002441, + "mean_logprob": -5.248795032501221, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -7.808121681213379, + "mean_logprob": -2.6027072270711265, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.07054901123046875, + "context_truncated": false, + "elapsed_seconds": 0.026309938000849797 + }, + { + "id": "context_tracking-033", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 1.7999999999999998, + "item_elo": 950, + "context": "For this report, green means urgent and red means routine. A green item is therefore", + "continuations": [ + " urgent.", + " routine.", + " canceled.", + " unclassified." + ], + "tags": [ + "remapping" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -8.035861015319824, + "mean_logprob": -2.6786203384399414, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.059852600097656, + "mean_logprob": -5.529926300048828, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -15.928162574768066, + "mean_logprob": -3.9820406436920166, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -17.694698333740234, + "mean_logprob": -4.423674583435059, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 1.3034203052520752, + "context_truncated": false, + "elapsed_seconds": 0.026921026001218706 + }, + { + "id": "context_tracking-034", + "category": "context_tracking", + "difficulty": "medium", + "category_weight": 1.2, + "difficulty_weight": 1.5, + "point_weight": 1.7999999999999998, + "earned_points": 0.0, + "item_elo": 950, + "context": "The oak is east of the pond. The bench is east of the oak. Relative to the pond, the bench is", + "continuations": [ + " west.", + " east.", + " north.", + " at the same place." + ], + "tags": [ + "spatial" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -8.630821228027344, + "mean_logprob": -4.315410614013672, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -8.045670509338379, + "mean_logprob": -4.0228352546691895, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -7.104247093200684, + "mean_logprob": -3.552123546600342, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -17.643739700317383, + "mean_logprob": -3.5287479400634765, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.023375606536865323, + "context_truncated": false, + "elapsed_seconds": 0.028784793001250364 + }, + { + "id": "context_tracking-035", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Four files are ordered J, K, L, M. K is moved after M, and J is then moved between L and M. The resulting order is", + "continuations": [ + " J, L, M, K.", + " L, M, J, K.", + " L, J, M, K.", + " M, L, J, K." + ], + "tags": [ + "state_update" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -5.9900970458984375, + "mean_logprob": -0.7487621307373047, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -8.535561561584473, + "mean_logprob": -1.066945195198059, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -8.659443855285645, + "mean_logprob": -1.0824304819107056, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -11.459959030151367, + "mean_logprob": -1.432494878768921, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 0.3181830644607544, + "context_truncated": false, + "elapsed_seconds": 0.046220570999139454 + }, + { + "id": "context_tracking-036", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "In a codebook, tulip means north, north means copper, and copper means seven. Following the chain from tulip leads to", + "continuations": [ + " north only.", + " tulip.", + " copper only.", + " seven." + ], + "tags": [ + "multi_hop" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -17.177152633666992, + "mean_logprob": -5.725717544555664, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -11.78298568725586, + "mean_logprob": -2.945746421813965, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -15.028417587280273, + "mean_logprob": -5.009472529093425, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -11.33794116973877, + "mean_logprob": -5.668970584869385, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 2.06372610727946, + "context_truncated": false, + "elapsed_seconds": 0.03617483699963486 + }, + { + "id": "context_tracking-037", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A cube starts in Bin R. It moves to S unless the flag is blue. The flag is blue, so it moves to T instead. Its final bin is", + "continuations": [ + " Bin T.", + " Bin S.", + " Bin R.", + " unknown." + ], + "tags": [ + "conditional_update" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -10.813809394836426, + "mean_logprob": -3.604603131612142, + "total_sequence_tokens": 36 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.571236610412598, + "mean_logprob": -2.523745536804199, + "total_sequence_tokens": 36 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -9.749302864074707, + "mean_logprob": -3.249767621358236, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -6.8441290855407715, + "mean_logprob": -3.4220645427703857, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.7260220845540366, + "context_truncated": false, + "elapsed_seconds": 0.03776685400043789 + }, + { + "id": "context_tracking-038", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Mira stands left of Noor. Olin stands right of Noor but left of Peta. From left to right, the second person is", + "continuations": [ + " Mira.", + " Noor.", + " Olin.", + " Peta." + ], + "tags": [ + "spatial" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -4.994318962097168, + "mean_logprob": -1.6647729873657227, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -4.311159610748291, + "mean_logprob": -1.4370532035827637, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -2.769989252090454, + "mean_logprob": -0.923329750696818, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -3.6630380153656006, + "mean_logprob": -1.2210126717885335, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.29768292109171546, + "context_truncated": false, + "elapsed_seconds": 0.034288531000129296 + }, + { + "id": "context_tracking-039", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "The archive uses rotating labels. On day one A means red; on day two A means blue; on day three A means gold. Today is day three, so A means", + "continuations": [ + " red.", + " blue.", + " gold.", + " every color." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -4.996253967285156, + "mean_logprob": -2.498126983642578, + "total_sequence_tokens": 37 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -4.155964374542236, + "mean_logprob": -2.077982187271118, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -0.3302699327468872, + "mean_logprob": -0.1651349663734436, + "total_sequence_tokens": 37 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -9.863689422607422, + "mean_logprob": -3.287896474202474, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 1.9128472208976746, + "context_truncated": false, + "elapsed_seconds": 0.04565987499881885 + }, + { + "id": "context_tracking-040", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A coin begins with Lio, passes to Mina, returns to Lio, then passes to Niko. Niko gives it to the person who held it immediately before him. The coin ends with", + "continuations": [ + " Mina.", + " Niko.", + " nobody.", + " Lio." + ], + "tags": [ + "state_update" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -1.7303788661956787, + "mean_logprob": -0.5767929553985596, + "total_sequence_tokens": 46 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -1.7619781494140625, + "mean_logprob": -0.4404945373535156, + "total_sequence_tokens": 47 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -12.21472454071045, + "mean_logprob": -4.071574846903483, + "total_sequence_tokens": 46 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -2.2031490802764893, + "mean_logprob": -0.7343830267588297, + "total_sequence_tokens": 46 + } + ], + "best_vs_second_margin": 0.13629841804504395, + "context_truncated": false, + "elapsed_seconds": 0.04590733899931365 + }, + { + "id": "context_tracking-041", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "In a race, Uma finished before Vic. Wes finished after Xena but before Uma. The earliest among Uma, Vic, Wes, and Xena was", + "continuations": [ + " Xena.", + " Wes.", + " Uma.", + " Vic." + ], + "tags": [ + "transitive" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -2.8171894550323486, + "mean_logprob": -0.9390631516774496, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -3.756889820098877, + "mean_logprob": -1.2522966066996257, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -3.0636844635009766, + "mean_logprob": -1.0212281545003254, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -2.9644932746887207, + "mean_logprob": -0.9881644248962402, + "total_sequence_tokens": 41 + } + ], + "best_vs_second_margin": 0.04910127321879065, + "context_truncated": false, + "elapsed_seconds": 0.04572442499920726 + }, + { + "id": "context_tracking-042", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "A ledger maps 4 to pine, pine to west, and west to 19. Looking up 4 repeatedly until reaching a number returns", + "continuations": [ + " west.", + " 19.", + " pine.", + " 4." + ], + "tags": [ + "multi_hop" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.747142791748047, + "mean_logprob": -5.873571395874023, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -4.2428412437438965, + "mean_logprob": -1.0607103109359741, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -9.676970481872559, + "mean_logprob": -4.838485240936279, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -6.3904032707214355, + "mean_logprob": -2.130134423573812, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 1.0694241126378379, + "context_truncated": false, + "elapsed_seconds": 0.03764528200008499 + }, + { + "id": "context_tracking-043", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "The black card outranks the white card. The striped card ranks below white but above gray. The second-highest card is", + "continuations": [ + " black.", + " striped.", + " white.", + " gray." + ], + "tags": [ + "order" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -3.5012102127075195, + "mean_logprob": -1.7506051063537598, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -8.898595809936523, + "mean_logprob": -2.9661986033121743, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -4.047337532043457, + "mean_logprob": -2.0236687660217285, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -5.855133056640625, + "mean_logprob": -2.9275665283203125, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.27306365966796875, + "context_truncated": false, + "elapsed_seconds": 0.03667803600001207 + }, + { + "id": "context_tracking-044", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "A marker is in Drawer A. Rule one moves it to B; rule two swaps the contents of B and C; rule three moves whatever is in C to D. The marker finishes in", + "continuations": [ + " Drawer A.", + " Drawer B.", + " Drawer C.", + " Drawer D." + ], + "tags": [ + "state_update" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -4.155341148376465, + "mean_logprob": -1.0388352870941162, + "total_sequence_tokens": 45 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -3.6743602752685547, + "mean_logprob": -0.9185900688171387, + "total_sequence_tokens": 45 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -5.7672014236450195, + "mean_logprob": -1.4418003559112549, + "total_sequence_tokens": 45 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -4.906957626342773, + "mean_logprob": -1.2267394065856934, + "total_sequence_tokens": 45 + } + ], + "best_vs_second_margin": 0.12024521827697754, + "context_truncated": false, + "elapsed_seconds": 0.04519589900155552 + }, + { + "id": "context_tracking-045", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "Q is north of R. S is east of Q. T is south of S and level with R. Relative to R, T is", + "continuations": [ + " east.", + " west.", + " north.", + " at the same position." + ], + "tags": [ + "spatial" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -10.444129943847656, + "mean_logprob": -5.222064971923828, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -9.060394287109375, + "mean_logprob": -4.5301971435546875, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -9.162068367004395, + "mean_logprob": -4.581034183502197, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -17.914993286132812, + "mean_logprob": -3.5829986572265624, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.9471984863281251, + "context_truncated": false, + "elapsed_seconds": 0.03621936600029585 + }, + { + "id": "context_tracking-046", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "There are three switches. Flipping X toggles Y; flipping Y toggles Z. Initially all are off. X is flipped once, then Y is flipped once. At the end, Z is", + "continuations": [ + " unchanged and off.", + " on.", + " removed.", + " both on and off." + ], + "tags": [ + "state_update" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -18.077299118041992, + "mean_logprob": -4.519324779510498, + "total_sequence_tokens": 47 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -1.6058751344680786, + "mean_logprob": -0.8029375672340393, + "total_sequence_tokens": 45 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -11.36464786529541, + "mean_logprob": -5.682323932647705, + "total_sequence_tokens": 45 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -8.00474739074707, + "mean_logprob": -1.600949478149414, + "total_sequence_tokens": 48 + } + ], + "best_vs_second_margin": 0.7980119109153747, + "context_truncated": false, + "elapsed_seconds": 0.045616373001394095 + }, + { + "id": "context_tracking-047", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "A roster initially reads Arin, Bela, Ciro, Dena. Bela and Dena swap places; then the first and third names swap. The first name becomes", + "continuations": [ + " Arin.", + " Dena.", + " Ciro.", + " Bela." + ], + "tags": [ + "permutation" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -3.308192253112793, + "mean_logprob": -1.1027307510375977, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -5.120464324951172, + "mean_logprob": -1.7068214416503906, + "total_sequence_tokens": 40 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -3.036705255508423, + "mean_logprob": -1.0122350851694744, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -7.030072212219238, + "mean_logprob": -2.3433574040730796, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 0.0904956658681233, + "context_truncated": false, + "elapsed_seconds": 0.03578504999859433 + }, + { + "id": "context_tracking-048", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "On odd dates, Gate P leads to Hall 1; on even dates it leads to Hall 2. Gate Q does the opposite. The date is the 18th, and a visitor enters Gate Q, reaching", + "continuations": [ + " Hall 2.", + " both halls.", + " no hall.", + " Hall 1." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -2.4496448040008545, + "mean_logprob": -0.6124112010002136, + "total_sequence_tokens": 52 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -12.987076759338379, + "mean_logprob": -3.2467691898345947, + "total_sequence_tokens": 52 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -17.982742309570312, + "mean_logprob": -5.9942474365234375, + "total_sequence_tokens": 51 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -0.7162995934486389, + "mean_logprob": -0.17907489836215973, + "total_sequence_tokens": 52 + } + ], + "best_vs_second_margin": 0.4333363026380539, + "context_truncated": false, + "elapsed_seconds": 0.047340212999188225 + }, + { + "id": "context_tracking-049", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 2.6999999999999997, + "item_elo": 1100, + "context": "Nora is assigned to Vega unless Ishan takes Vega; if he does, Nora takes Lyra. Ishan takes Vega. Nora is therefore assigned to", + "continuations": [ + " Lyra.", + " Vega.", + " both.", + " neither." + ], + "tags": [ + "conditional_binding" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -1.1825419664382935, + "mean_logprob": -0.39418065547943115, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -1.770630121231079, + "mean_logprob": -0.5902100404103597, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -9.81666088104248, + "mean_logprob": -4.90833044052124, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -13.6995849609375, + "mean_logprob": -6.84979248046875, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 0.19602938493092859, + "context_truncated": false, + "elapsed_seconds": 0.03791658200134407 + }, + { + "id": "context_tracking-050", + "category": "context_tracking", + "difficulty": "hard", + "category_weight": 1.2, + "difficulty_weight": 2.25, + "point_weight": 2.6999999999999997, + "earned_points": 0.0, + "item_elo": 1100, + "context": "The sequence begins amber, cobalt, jade, silver. Remove cobalt, place silver first, then insert rose immediately after amber. The final sequence is", + "continuations": [ + " silver, rose, amber, jade.", + " silver, amber, rose, jade.", + " amber, rose, silver, jade.", + " silver, amber, jade, rose." + ], + "tags": [ + "permutation" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -9.478717803955078, + "mean_logprob": -0.9478717803955078, + "total_sequence_tokens": 46 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -9.959565162658691, + "mean_logprob": -0.9959565162658691, + "total_sequence_tokens": 46 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -11.633020401000977, + "mean_logprob": -1.1633020401000977, + "total_sequence_tokens": 46 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -15.396784782409668, + "mean_logprob": -1.5396784782409667, + "total_sequence_tokens": 46 + } + ], + "best_vs_second_margin": 0.04808473587036133, + "context_truncated": false, + "elapsed_seconds": 0.044577638000191655 + }, + { + "id": "quantitative-001", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Seven plus five equals", + "continuations": [ + " twelve.", + " eleven.", + " thirteen.", + " fourteen." + ], + "tags": [ + "arithmetic" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -5.292641639709473, + "mean_logprob": -2.6463208198547363, + "total_sequence_tokens": 7 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -5.131879806518555, + "mean_logprob": -1.7106266021728516, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -6.535145282745361, + "mean_logprob": -2.1783817609151206, + "total_sequence_tokens": 8 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -7.454742431640625, + "mean_logprob": -2.4849141438802085, + "total_sequence_tokens": 8 + } + ], + "best_vs_second_margin": 0.46775515874226903, + "context_truncated": false, + "elapsed_seconds": 0.033248167999772704 + }, + { + "id": "quantitative-002", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A box holds nine pencils. After three are removed, it holds", + "continuations": [ + " five pencils.", + " twelve pencils.", + " seven pencils.", + " six pencils." + ], + "tags": [ + "subtraction" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -3.5479001998901367, + "mean_logprob": -1.182633399963379, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -3.5878844261169434, + "mean_logprob": -1.1959614753723145, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -4.939874649047852, + "mean_logprob": -1.6466248830159504, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -4.488526344299316, + "mean_logprob": -1.4961754480997722, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.013328075408935547, + "context_truncated": false, + "elapsed_seconds": 0.02531888900011836 + }, + { + "id": "quantitative-003", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Four groups of three objects contain", + "continuations": [ + " seven objects.", + " ten objects.", + " twelve objects.", + " sixteen objects." + ], + "tags": [ + "multiplication" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -11.508077621459961, + "mean_logprob": -3.836025873819987, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -10.469986915588379, + "mean_logprob": -3.4899956385294595, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -13.069352149963379, + "mean_logprob": -4.35645071665446, + "total_sequence_tokens": 9 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -13.693161964416504, + "mean_logprob": -3.423290491104126, + "total_sequence_tokens": 10 + } + ], + "best_vs_second_margin": 0.06670514742533351, + "context_truncated": false, + "elapsed_seconds": 0.028185780000058003 + }, + { + "id": "quantitative-004", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "Twenty divided equally among five people gives each person", + "continuations": [ + " two.", + " three.", + " five.", + " four." + ], + "tags": [ + "division" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -8.99081802368164, + "mean_logprob": -4.49540901184082, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -8.719918251037598, + "mean_logprob": -4.359959125518799, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -9.545299530029297, + "mean_logprob": -4.772649765014648, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -7.664897918701172, + "mean_logprob": -3.832448959350586, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.5275101661682129, + "context_truncated": false, + "elapsed_seconds": 0.025594724000256974 + }, + { + "id": "quantitative-005", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "The number immediately after 199 is", + "continuations": [ + " 200.", + " 198.", + " 209.", + " 299." + ], + "tags": [ + "number_sense" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -2.5621232986450195, + "mean_logprob": -0.5124246597290039, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -5.487677574157715, + "mean_logprob": -1.0975355148315429, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -8.877996444702148, + "mean_logprob": -1.7755992889404297, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -6.280266761779785, + "mean_logprob": -1.256053352355957, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.585110855102539, + "context_truncated": false, + "elapsed_seconds": 0.024640380001073936 + }, + { + "id": "quantitative-006", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Half of eighteen is", + "continuations": [ + " eight.", + " nine.", + " ten.", + " thirty-six." + ], + "tags": [ + "fractions" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -5.813625812530518, + "mean_logprob": -2.906812906265259, + "total_sequence_tokens": 7 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -6.29595947265625, + "mean_logprob": -3.147979736328125, + "total_sequence_tokens": 7 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -6.812882423400879, + "mean_logprob": -3.4064412117004395, + "total_sequence_tokens": 7 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -8.896859169006348, + "mean_logprob": -2.224214792251587, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 0.6825981140136719, + "context_truncated": false, + "elapsed_seconds": 0.02822259400090843 + }, + { + "id": "quantitative-007", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A rectangle with length five and width two has area", + "continuations": [ + " seven.", + " twelve.", + " ten.", + " fourteen." + ], + "tags": [ + "geometry" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -6.672070503234863, + "mean_logprob": -3.3360352516174316, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -8.611666679382324, + "mean_logprob": -4.305833339691162, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -7.9228715896606445, + "mean_logprob": -3.9614357948303223, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -7.710935592651367, + "mean_logprob": -2.5703118642171225, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.7657233874003091, + "context_truncated": false, + "elapsed_seconds": 0.02752333599892154 + }, + { + "id": "quantitative-008", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "Three quarters of an hour is", + "continuations": [ + " thirty minutes.", + " forty minutes.", + " sixty minutes.", + " forty-five minutes." + ], + "tags": [ + "time" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -8.348872184753418, + "mean_logprob": -2.782957394917806, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -9.526144027709961, + "mean_logprob": -3.175381342569987, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -10.311846733093262, + "mean_logprob": -2.0623693466186523, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -9.686224937438965, + "mean_logprob": -1.937244987487793, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.1251243591308593, + "context_truncated": false, + "elapsed_seconds": 0.03023112299888453 + }, + { + "id": "quantitative-009", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A ten-euro item discounted by two euros costs", + "continuations": [ + " eight euros.", + " twelve euros.", + " five euros.", + " twenty euros." + ], + "tags": [ + "money" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -10.715323448181152, + "mean_logprob": -2.1430646896362306, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -13.067676544189453, + "mean_logprob": -2.6135353088378905, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -9.866440773010254, + "mean_logprob": -1.9732881546020509, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -11.636058807373047, + "mean_logprob": -2.3272117614746093, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.16977653503417978, + "context_truncated": false, + "elapsed_seconds": 0.02641895899978408 + }, + { + "id": "quantitative-010", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "The perimeter of a square with side length three is", + "continuations": [ + " nine.", + " twelve.", + " six.", + " fifteen." + ], + "tags": [ + "geometry" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.847151756286621, + "mean_logprob": -5.9235758781433105, + "total_sequence_tokens": 12 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.464617729187012, + "mean_logprob": -5.732308864593506, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -11.502902030944824, + "mean_logprob": -5.751451015472412, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -11.837128639221191, + "mean_logprob": -5.918564319610596, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.01914215087890625, + "context_truncated": false, + "elapsed_seconds": 0.02509022899903357 + }, + { + "id": "quantitative-011", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "Thirty is five more than", + "continuations": [ + " twenty.", + " thirty-five.", + " twenty-five.", + " fifteen." + ], + "tags": [ + "arithmetic" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -4.370925426483154, + "mean_logprob": -2.185462713241577, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -8.961140632629395, + "mean_logprob": -2.2402851581573486, + "total_sequence_tokens": 11 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -6.706611156463623, + "mean_logprob": -1.6766527891159058, + "total_sequence_tokens": 11 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -6.271145820617676, + "mean_logprob": -3.135572910308838, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 0.5088099241256714, + "context_truncated": false, + "elapsed_seconds": 0.027746277999540325 + }, + { + "id": "quantitative-012", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 1.3, + "item_elo": 900, + "context": "If one notebook costs four euros, two notebooks cost", + "continuations": [ + " four euros.", + " six euros.", + " twelve euros.", + " eight euros." + ], + "tags": [ + "money" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -5.146291255950928, + "mean_logprob": -1.0292582511901855, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -5.293211936950684, + "mean_logprob": -1.0586423873901367, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -7.491818904876709, + "mean_logprob": -1.4983637809753418, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -4.346880912780762, + "mean_logprob": -0.8693761825561523, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.1598820686340332, + "context_truncated": false, + "elapsed_seconds": 0.026342607001424767 + }, + { + "id": "quantitative-013", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "The difference between fifteen and nine is", + "continuations": [ + " six.", + " five.", + " seven.", + " twenty-four." + ], + "tags": [ + "subtraction" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -5.347989082336426, + "mean_logprob": -2.673994541168213, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -4.929468631744385, + "mean_logprob": -2.4647343158721924, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -4.034341812133789, + "mean_logprob": -2.0171709060668945, + "total_sequence_tokens": 9 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -9.035197257995605, + "mean_logprob": -2.2587993144989014, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.24162840843200684, + "context_truncated": false, + "elapsed_seconds": 0.025635102998421644 + }, + { + "id": "quantitative-014", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Counting by tens, the number after forty is", + "continuations": [ + " forty-one.", + " fifty.", + " sixty.", + " thirty." + ], + "tags": [ + "number_sense" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -6.080322265625, + "mean_logprob": -1.52008056640625, + "total_sequence_tokens": 14 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -4.013950347900391, + "mean_logprob": -2.0069751739501953, + "total_sequence_tokens": 12 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -7.232993125915527, + "mean_logprob": -1.8082482814788818, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -2.881405830383301, + "mean_logprob": -1.4407029151916504, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.07937765121459961, + "context_truncated": false, + "elapsed_seconds": 0.025576610998541582 + }, + { + "id": "quantitative-015", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "Two meters equal", + "continuations": [ + " twenty centimeters.", + " two hundred millimeters.", + " two hundred centimeters.", + " two thousand centimeters." + ], + "tags": [ + "units" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -14.974717140197754, + "mean_logprob": -4.991572380065918, + "total_sequence_tokens": 6 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -16.25657081604004, + "mean_logprob": -3.251314163208008, + "total_sequence_tokens": 8 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -15.64132308959961, + "mean_logprob": -3.9103307723999023, + "total_sequence_tokens": 7 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -17.949125289916992, + "mean_logprob": -4.487281322479248, + "total_sequence_tokens": 7 + } + ], + "best_vs_second_margin": 0.6590166091918945, + "context_truncated": false, + "elapsed_seconds": 0.027211262000491843 + }, + { + "id": "quantitative-016", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "The average of four and eight is", + "continuations": [ + " four.", + " five.", + " eight.", + " six." + ], + "tags": [ + "average" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -5.802121162414551, + "mean_logprob": -2.9010605812072754, + "total_sequence_tokens": 9 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -6.14479398727417, + "mean_logprob": -3.072396993637085, + "total_sequence_tokens": 9 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -6.672871112823486, + "mean_logprob": -3.336435556411743, + "total_sequence_tokens": 9 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -6.14555025100708, + "mean_logprob": -3.07277512550354, + "total_sequence_tokens": 9 + } + ], + "best_vs_second_margin": 0.17133641242980957, + "context_truncated": false, + "elapsed_seconds": 0.02484983700014709 + }, + { + "id": "quantitative-017", + "category": "quantitative", + "difficulty": "easy", + "category_weight": 1.3, + "difficulty_weight": 1.0, + "point_weight": 1.3, + "earned_points": 0.0, + "item_elo": 900, + "context": "A dozen eggs contains", + "continuations": [ + " twelve eggs.", + " ten eggs.", + " six eggs.", + " twenty eggs." + ], + "tags": [ + "number_sense" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -9.102995872497559, + "mean_logprob": -3.034331957499186, + "total_sequence_tokens": 7 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.773576259613037, + "mean_logprob": -2.591192086537679, + "total_sequence_tokens": 7 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -8.962491035461426, + "mean_logprob": -2.987497011820475, + "total_sequence_tokens": 7 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -8.10993480682373, + "mean_logprob": -2.7033116022745767, + "total_sequence_tokens": 7 + } + ], + "best_vs_second_margin": 0.11211951573689749, + "context_truncated": false, + "elapsed_seconds": 0.024494163000781555 + }, + { + "id": "quantitative-018", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A train travels 180 kilometers in three hours at a constant rate. Its speed is", + "continuations": [ + " forty kilometers per hour.", + " sixty kilometers per hour.", + " ninety kilometers per hour.", + " one hundred eighty kilometers per hour." + ], + "tags": [ + "rate" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -16.961780548095703, + "mean_logprob": -3.392356109619141, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -26.406064987182617, + "mean_logprob": -3.7722949981689453, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -17.92763900756836, + "mean_logprob": -2.56109128679548, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -23.113128662109375, + "mean_logprob": -2.889141082763672, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.32804979596819184, + "context_truncated": false, + "elapsed_seconds": 0.029687533000469557 + }, + { + "id": "quantitative-019", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A jacket priced at 80 euros is reduced by 25 percent. The sale price is", + "continuations": [ + " fifty euros.", + " fifty-five euros.", + " sixty euros.", + " sixty-five euros." + ], + "tags": [ + "percent" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -18.02532386779785, + "mean_logprob": -3.6050647735595702, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -19.415939331054688, + "mean_logprob": -2.773705618722098, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -17.058008193969727, + "mean_logprob": -2.4368583134242465, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -13.648637771606445, + "mean_logprob": -1.5165153079562717, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.9203430054679749, + "context_truncated": false, + "elapsed_seconds": 0.04079642699980468 + }, + { + "id": "quantitative-020", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "A tank contains 36 liters. One third is drained, leaving", + "continuations": [ + " twelve liters.", + " eighteen liters.", + " thirty liters.", + " twenty-four liters." + ], + "tags": [ + "fractions" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -14.342558860778809, + "mean_logprob": -4.7808529535929365, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -16.174556732177734, + "mean_logprob": -4.043639183044434, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -16.89756202697754, + "mean_logprob": -5.63252067565918, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -15.412789344787598, + "mean_logprob": -3.0825578689575197, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.9610813140869139, + "context_truncated": false, + "elapsed_seconds": 0.045352904999163 + }, + { + "id": "quantitative-021", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Five consecutive integers begin with 12. Their sum is", + "continuations": [ + " seventy.", + " sixty.", + " seventy-five.", + " eighty." + ], + "tags": [ + "sequence" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -17.021854400634766, + "mean_logprob": -5.673951466878255, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -23.48359489440918, + "mean_logprob": -5.870898723602295, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -14.351556777954102, + "mean_logprob": -2.8703113555908204, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -17.526702880859375, + "mean_logprob": -5.842234293619792, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 2.8036401112874345, + "context_truncated": false, + "elapsed_seconds": 0.031220798000504146 + }, + { + "id": "quantitative-022", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A right triangle has legs of length 6 and 8. Its hypotenuse has length", + "continuations": [ + " nine.", + " ten.", + " twelve.", + " fourteen." + ], + "tags": [ + "geometry" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.915306091308594, + "mean_logprob": -5.957653045654297, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -13.793683052062988, + "mean_logprob": -6.896841526031494, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.201858520507812, + "mean_logprob": -6.600929260253906, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -18.208065032958984, + "mean_logprob": -6.069355010986328, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.11170196533203125, + "context_truncated": false, + "elapsed_seconds": 0.027931208000154584 + }, + { + "id": "quantitative-023", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "If x plus 7 equals 19, then x equals", + "continuations": [ + " ten.", + " eleven.", + " twelve.", + " twenty-six." + ], + "tags": [ + "algebra" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.407376289367676, + "mean_logprob": -5.703688144683838, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -13.684794425964355, + "mean_logprob": -4.561598141988118, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -13.927817344665527, + "mean_logprob": -6.963908672332764, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -15.525569915771484, + "mean_logprob": -3.881392478942871, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.6802056630452471, + "context_truncated": false, + "elapsed_seconds": 0.03209438700105238 + }, + { + "id": "quantitative-024", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "A worker earns 18 euros per hour for seven hours. Total earnings are", + "continuations": [ + " 108 euros.", + " 116 euros.", + " 136 euros.", + " 126 euros." + ], + "tags": [ + "multiplication" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -9.774751663208008, + "mean_logprob": -1.221843957901001, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -10.015142440795898, + "mean_logprob": -1.2518928050994873, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -11.240349769592285, + "mean_logprob": -1.4050437211990356, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -8.623146057128906, + "mean_logprob": -1.0778932571411133, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.1439507007598877, + "context_truncated": false, + "elapsed_seconds": 0.06568012900061149 + }, + { + "id": "quantitative-025", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "The ratio of red to blue marbles is 3 to 5. If there are 15 red marbles, there are", + "continuations": [ + " twenty-five blue marbles.", + " nine blue marbles.", + " eighteen blue marbles.", + " seventy-five blue marbles." + ], + "tags": [ + "ratio" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -16.056371688842773, + "mean_logprob": -2.6760619481404624, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -12.535652160644531, + "mean_logprob": -3.133913040161133, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -17.886301040649414, + "mean_logprob": -3.577260208129883, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -20.623779296875, + "mean_logprob": -2.946254185267857, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.2701922371273948, + "context_truncated": false, + "elapsed_seconds": 0.04084540600160835 + }, + { + "id": "quantitative-026", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A circle has radius 4. Using pi as 3.14, its circumference is approximately", + "continuations": [ + " 12.56.", + " 25.12.", + " 50.24.", + " 16.00." + ], + "tags": [ + "geometry" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -3.247330665588379, + "mean_logprob": -0.46390438079833984, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -11.988222122192383, + "mean_logprob": -1.7126031603131975, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -10.607452392578125, + "mean_logprob": -1.515350341796875, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -26.597305297851562, + "mean_logprob": -3.799615042550223, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 1.0514459609985352, + "context_truncated": false, + "elapsed_seconds": 0.026777819999551866 + }, + { + "id": "quantitative-027", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "An account balance rises from 240 to 300. The percentage increase is", + "continuations": [ + " fifteen percent.", + " twenty percent.", + " twenty-five percent.", + " sixty percent." + ], + "tags": [ + "percent" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -19.058584213256836, + "mean_logprob": -6.352861404418945, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -19.500022888183594, + "mean_logprob": -6.500007629394531, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -18.61857795715332, + "mean_logprob": -3.723715591430664, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -19.820444107055664, + "mean_logprob": -3.9640888214111327, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.24037322998046884, + "context_truncated": false, + "elapsed_seconds": 0.02628384899981029 + }, + { + "id": "quantitative-028", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A bus leaves at 09:35 and arrives at 11:05. The journey lasts", + "continuations": [ + " seventy minutes.", + " eighty minutes.", + " one hundred minutes.", + " ninety minutes." + ], + "tags": [ + "time" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -19.287830352783203, + "mean_logprob": -4.821957588195801, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -33.98118591308594, + "mean_logprob": -8.495296478271484, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -19.279085159301758, + "mean_logprob": -4.8197712898254395, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -30.71573829650879, + "mean_logprob": -6.143147659301758, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.002186298370361328, + "context_truncated": false, + "elapsed_seconds": 0.02756844000032288 + }, + { + "id": "quantitative-029", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "The median of 3, 7, 8, 12, and 20 is", + "continuations": [ + " eight.", + " seven.", + " ten.", + " twelve." + ], + "tags": [ + "statistics" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -12.059478759765625, + "mean_logprob": -6.0297393798828125, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -12.399323463439941, + "mean_logprob": -6.199661731719971, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -14.319886207580566, + "mean_logprob": -7.159943103790283, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -14.025663375854492, + "mean_logprob": -7.012831687927246, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.1699223518371582, + "context_truncated": false, + "elapsed_seconds": 0.05327712399957818 + }, + { + "id": "quantitative-030", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "A 48-page booklet is printed with four pages per sheet. It requires", + "continuations": [ + " ten sheets.", + " twelve sheets.", + " sixteen sheets.", + " forty-four sheets." + ], + "tags": [ + "division" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -11.124528884887695, + "mean_logprob": -3.708176294962565, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -10.980240821838379, + "mean_logprob": -3.6600802739461265, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -12.329124450683594, + "mean_logprob": -3.0822811126708984, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -16.06705093383789, + "mean_logprob": -3.213410186767578, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.1311290740966795, + "context_truncated": false, + "elapsed_seconds": 0.05290328900082386 + }, + { + "id": "quantitative-031", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "Three fifths written as a decimal is", + "continuations": [ + " 0.3.", + " 0.5.", + " 0.6.", + " 0.8." + ], + "tags": [ + "fractions" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -7.394290924072266, + "mean_logprob": -1.478858184814453, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -4.609033107757568, + "mean_logprob": -0.9218066215515137, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -5.725163459777832, + "mean_logprob": -1.1450326919555665, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -4.468921661376953, + "mean_logprob": -0.8937843322753907, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.028022289276123047, + "context_truncated": false, + "elapsed_seconds": 0.025549125999532407 + }, + { + "id": "quantitative-032", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 1.9500000000000002, + "item_elo": 1000, + "context": "A shop buys an item for 30 euros and sells it for 42 euros. The profit is", + "continuations": [ + " 10 euros.", + " 11 euros.", + " 72 euros.", + " 12 euros." + ], + "tags": [ + "money" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -3.9913277626037598, + "mean_logprob": -0.5701896803719657, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -6.558568954467773, + "mean_logprob": -0.9369384220668248, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -8.563762664794922, + "mean_logprob": -1.2233946663992745, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -3.681147336959839, + "mean_logprob": -0.5258781909942627, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.04431148937770302, + "context_truncated": false, + "elapsed_seconds": 0.03577348799990432 + }, + { + "id": "quantitative-033", + "category": "quantitative", + "difficulty": "medium", + "category_weight": 1.3, + "difficulty_weight": 1.5, + "point_weight": 1.9500000000000002, + "earned_points": 0.0, + "item_elo": 1000, + "context": "If 2x minus 4 equals 16, then x equals", + "continuations": [ + " ten.", + " six.", + " eight.", + " twelve." + ], + "tags": [ + "algebra" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -10.682738304138184, + "mean_logprob": -5.341369152069092, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -10.106553077697754, + "mean_logprob": -5.053276538848877, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -10.52470874786377, + "mean_logprob": -5.262354373931885, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -10.924354553222656, + "mean_logprob": -5.462177276611328, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.2090778350830078, + "context_truncated": false, + "elapsed_seconds": 0.03075260699915816 + }, + { + "id": "quantitative-034", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A price increases by 20 percent and is then reduced by 20 percent. Starting from 100 euros, the final price is", + "continuations": [ + " 100 euros.", + " 96 euros.", + " 80 euros.", + " 104 euros." + ], + "tags": [ + "percent" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -14.256305694580078, + "mean_logprob": -1.7820382118225098, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -10.194211959838867, + "mean_logprob": -1.4563159942626953, + "total_sequence_tokens": 39 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -7.444924831390381, + "mean_logprob": -1.0635606901986259, + "total_sequence_tokens": 39 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -11.327791213989258, + "mean_logprob": -1.4159739017486572, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 0.35241321155003136, + "context_truncated": false, + "elapsed_seconds": 0.0364853069986566 + }, + { + "id": "quantitative-035", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "A mixture contains milk and water in a 5 to 2 ratio. In 21 liters of mixture, the amount of water is", + "continuations": [ + " five liters.", + " seven liters.", + " six liters.", + " fifteen liters." + ], + "tags": [ + "ratio" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -14.469293594360352, + "mean_logprob": -4.823097864786784, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -16.154003143310547, + "mean_logprob": -5.384667714436849, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -13.7432279586792, + "mean_logprob": -4.581075986226399, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -15.343059539794922, + "mean_logprob": -5.114353179931641, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.2420218785603847, + "context_truncated": false, + "elapsed_seconds": 0.036742966000019806 + }, + { + "id": "quantitative-036", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "Two numbers have sum 46 and difference 8. The larger number is", + "continuations": [ + " nineteen.", + " twenty-three.", + " thirty-five.", + " twenty-seven." + ], + "tags": [ + "algebra" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -20.145109176635742, + "mean_logprob": -5.0362772941589355, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -15.645545959472656, + "mean_logprob": -3.911386489868164, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -14.99804973602295, + "mean_logprob": -3.7495124340057373, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -14.937511444091797, + "mean_logprob": -3.734377861022949, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.015134572982788086, + "context_truncated": false, + "elapsed_seconds": 0.027836683999339584 + }, + { + "id": "quantitative-037", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A car travels the first 120 kilometers at 60 km/h and the next 120 at 40 km/h. Its average speed for the full trip is", + "continuations": [ + " 48 km/h.", + " 50 km/h.", + " 45 km/h.", + " 52 km/h." + ], + "tags": [ + "rate" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -6.520872116088867, + "mean_logprob": -0.9315531594412667, + "total_sequence_tokens": 47 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -3.3077588081359863, + "mean_logprob": -0.4725369725908552, + "total_sequence_tokens": 47 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -0.6295626163482666, + "mean_logprob": -0.08993751662118095, + "total_sequence_tokens": 47 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -4.594418048858643, + "mean_logprob": -0.6563454355512347, + "total_sequence_tokens": 47 + } + ], + "best_vs_second_margin": 0.38259945596967426, + "context_truncated": false, + "elapsed_seconds": 0.03457150299982459 + }, + { + "id": "quantitative-038", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A rectangle's length is three more than its width. If its perimeter is 30, its width is", + "continuations": [ + " five.", + " six.", + " seven.", + " nine." + ], + "tags": [ + "algebra" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.429022789001465, + "mean_logprob": -5.714511394500732, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -11.57111930847168, + "mean_logprob": -5.78555965423584, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -12.157730102539062, + "mean_logprob": -6.078865051269531, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -13.113155364990234, + "mean_logprob": -6.556577682495117, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.07104825973510742, + "context_truncated": false, + "elapsed_seconds": 0.031772250000358326 + }, + { + "id": "quantitative-039", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A bag has four red, three blue, and five green balls. The probability of drawing a blue ball is", + "continuations": [ + " one third.", + " three fifths.", + " one quarter.", + " three quarters." + ], + "tags": [ + "probability" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -17.496288299560547, + "mean_logprob": -5.832096099853516, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -15.568428039550781, + "mean_logprob": -3.8921070098876953, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -16.224609375, + "mean_logprob": -5.408203125, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -17.390769958496094, + "mean_logprob": -5.796923319498698, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 1.5160961151123047, + "context_truncated": false, + "elapsed_seconds": 0.03168500199899427 + }, + { + "id": "quantitative-040", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "An investment of 2,000 euros earns simple interest at 4 percent per year for three years. The interest earned is", + "continuations": [ + " 80 euros.", + " 160 euros.", + " 2,240 euros.", + " 240 euros." + ], + "tags": [ + "finance" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -8.198927879333496, + "mean_logprob": -1.1712754113333566, + "total_sequence_tokens": 37 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -8.330878257751465, + "mean_logprob": -1.041359782218933, + "total_sequence_tokens": 38 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -17.7349910736084, + "mean_logprob": -1.77349910736084, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -8.385602951049805, + "mean_logprob": -1.0482003688812256, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 0.0068405866622924805, + "context_truncated": false, + "elapsed_seconds": 0.036835486000200035 + }, + { + "id": "quantitative-041", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "The sequence 2, 6, 12, 20, 30 continues with", + "continuations": [ + " 42.", + " 36.", + " 40.", + " 44." + ], + "tags": [ + "sequence" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -12.798002243041992, + "mean_logprob": -3.199500560760498, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -11.089204788208008, + "mean_logprob": -2.772301197052002, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -12.132857322692871, + "mean_logprob": -3.0332143306732178, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -12.851228713989258, + "mean_logprob": -3.2128071784973145, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.2609131336212158, + "context_truncated": false, + "elapsed_seconds": 0.030952609000451048 + }, + { + "id": "quantitative-042", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A machine completes three jobs in 18 minutes at a constant rate. At that rate, eight jobs take", + "continuations": [ + " 36 minutes.", + " 48 minutes.", + " 54 minutes.", + " 144 minutes." + ], + "tags": [ + "rate" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -4.16306209564209, + "mean_logprob": -0.832612419128418, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -4.029392242431641, + "mean_logprob": -0.8058784484863282, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -3.299407720565796, + "mean_logprob": -0.6598815441131591, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -7.349918842315674, + "mean_logprob": -1.224986473719279, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.14599690437316903, + "context_truncated": false, + "elapsed_seconds": 0.027485315999001614 + }, + { + "id": "quantitative-043", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "The mean of six numbers is 14. Five of them sum to 61. The sixth number is", + "continuations": [ + " fourteen.", + " seventeen.", + " twenty-three.", + " eighty-four." + ], + "tags": [ + "statistics" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -16.144824981689453, + "mean_logprob": -5.381608327229817, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -11.930179595947266, + "mean_logprob": -3.976726531982422, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -13.954082489013672, + "mean_logprob": -3.488520622253418, + "total_sequence_tokens": 27 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -19.000680923461914, + "mean_logprob": -3.800136184692383, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.31161556243896493, + "context_truncated": false, + "elapsed_seconds": 0.025930818999768235 + }, + { + "id": "quantitative-044", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A map scale is 1 to 50,000. A distance of 6 centimeters on the map represents", + "continuations": [ + " 300 meters.", + " 30 kilometers.", + " 300 kilometers.", + " 3 kilometers." + ], + "tags": [ + "scale" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -9.360454559326172, + "mean_logprob": -1.5600757598876953, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -9.09926986694336, + "mean_logprob": -1.8198539733886718, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -8.061050415039062, + "mean_logprob": -1.3435084025065105, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -7.9915056228637695, + "mean_logprob": -1.9978764057159424, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.21656735738118482, + "context_truncated": false, + "elapsed_seconds": 0.0256035399997927 + }, + { + "id": "quantitative-045", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "If three quarters of a number is 27, the number is", + "continuations": [ + " thirty-six.", + " twenty-one.", + " thirty.", + " forty." + ], + "tags": [ + "algebra" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -13.211942672729492, + "mean_logprob": -3.302985668182373, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -15.152857780456543, + "mean_logprob": -3.7882144451141357, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -15.172837257385254, + "mean_logprob": -7.586418628692627, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -13.899062156677246, + "mean_logprob": -6.949531078338623, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.4852287769317627, + "context_truncated": false, + "elapsed_seconds": 0.02752245900046546 + }, + { + "id": "quantitative-046", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A 15-liter solution is 40 percent salt. Adding 5 liters of water makes the new salt concentration", + "continuations": [ + " twenty percent.", + " thirty percent.", + " thirty-five percent.", + " forty-five percent." + ], + "tags": [ + "mixture" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -20.30893325805664, + "mean_logprob": -6.769644419352214, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -20.39630126953125, + "mean_logprob": -6.79876708984375, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -17.16763687133789, + "mean_logprob": -3.433527374267578, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -16.944379806518555, + "mean_logprob": -3.388875961303711, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.0446514129638671, + "context_truncated": false, + "elapsed_seconds": 0.029969453000376234 + }, + { + "id": "quantitative-047", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A square's diagonal has length 10. Its area is", + "continuations": [ + " twenty-five.", + " forty.", + " fifty.", + " one hundred." + ], + "tags": [ + "geometry" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -17.12101936340332, + "mean_logprob": -4.28025484085083, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -21.154712677001953, + "mean_logprob": -10.577356338500977, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -21.690587997436523, + "mean_logprob": -10.845293998718262, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -22.510478973388672, + "mean_logprob": -7.503492991129558, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 3.2232381502787275, + "context_truncated": false, + "elapsed_seconds": 0.03216763900127262 + }, + { + "id": "quantitative-048", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A clock gains two minutes every three hours. After 24 real hours, it will be ahead by", + "continuations": [ + " eight minutes.", + " twelve minutes.", + " eighteen minutes.", + " sixteen minutes." + ], + "tags": [ + "rate" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -6.064818382263184, + "mean_logprob": -2.021606127421061, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -6.02801513671875, + "mean_logprob": -2.00933837890625, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -9.82884693145752, + "mean_logprob": -2.45721173286438, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -8.988325119018555, + "mean_logprob": -2.2470812797546387, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.01226774851481105, + "context_truncated": false, + "elapsed_seconds": 0.031557569000142394 + }, + { + "id": "quantitative-049", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 0.0, + "item_elo": 1150, + "context": "A number is increased by 8, doubled, and then reduced by 6 to give 30. The original number is", + "continuations": [ + " ten.", + " eight.", + " twelve.", + " fourteen." + ], + "tags": [ + "algebra" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -11.160680770874023, + "mean_logprob": -5.580340385437012, + "total_sequence_tokens": 28 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -8.79175853729248, + "mean_logprob": -4.39587926864624, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -11.474509239196777, + "mean_logprob": -5.737254619598389, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -10.697616577148438, + "mean_logprob": -3.5658721923828125, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.8300070762634277, + "context_truncated": false, + "elapsed_seconds": 0.03413807600009022 + }, + { + "id": "quantitative-050", + "category": "quantitative", + "difficulty": "hard", + "category_weight": 1.3, + "difficulty_weight": 2.25, + "point_weight": 2.9250000000000003, + "earned_points": 2.9250000000000003, + "item_elo": 1150, + "context": "A store marks an item up 25 percent from a cost of 64 euros. The marked price is", + "continuations": [ + " 72 euros.", + " 80 euros.", + " 84 euros.", + " 89 euros." + ], + "tags": [ + "percent" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -4.643354415893555, + "mean_logprob": -0.6633363451276507, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -4.039778709411621, + "mean_logprob": -0.5771112442016602, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -6.562363147735596, + "mean_logprob": -0.9374804496765137, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -8.166106224060059, + "mean_logprob": -1.1665866034371513, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.08622510092599056, + "context_truncated": false, + "elapsed_seconds": 0.027850759999637376 + }, + { + "id": "logical_reasoning-001", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "All robins are birds. Pip is a robin. Therefore, Pip is", + "continuations": [ + " a bird.", + " not a bird.", + " a fish.", + " necessarily blue." + ], + "tags": [ + "deduction" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -5.120014190673828, + "mean_logprob": -1.7066713968912761, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -5.987151145935059, + "mean_logprob": -1.4967877864837646, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -5.111690044403076, + "mean_logprob": -1.7038966814676921, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -17.19256591796875, + "mean_logprob": -5.730855305989583, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.20710889498392748, + "context_truncated": false, + "elapsed_seconds": 0.028051785000570817 + }, + { + "id": "logical_reasoning-002", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "No squares are circles. Shape K is a square. Shape K is therefore", + "continuations": [ + " a circle.", + " not a square.", + " both shapes.", + " not a circle." + ], + "tags": [ + "deduction" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -2.540952205657959, + "mean_logprob": -0.846984068552653, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -5.215645790100098, + "mean_logprob": -1.3039114475250244, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -14.818357467651367, + "mean_logprob": -4.939452489217122, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -6.576274871826172, + "mean_logprob": -1.644068717956543, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.45692737897237146, + "context_truncated": false, + "elapsed_seconds": 0.026913691999652656 + }, + { + "id": "logical_reasoning-003", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "If the switch is on, the lamp glows. The switch is on, so the lamp", + "continuations": [ + " must be broken.", + " is off.", + " glows.", + " has no switch." + ], + "tags": [ + "modus_ponens" + ], + "prediction": 1, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -11.94361686706543, + "mean_logprob": -2.9859042167663574, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -2.789653778076172, + "mean_logprob": -0.9298845926920573, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -3.650395154953003, + "mean_logprob": -1.2167983849843342, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -19.535369873046875, + "mean_logprob": -4.883842468261719, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.286913792292277, + "context_truncated": false, + "elapsed_seconds": 0.027348768000592827 + }, + { + "id": "logical_reasoning-004", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "A is taller than B, and B is taller than C. It follows that", + "continuations": [ + " C is taller than A.", + " A and C are equal.", + " B is shortest.", + " A is taller than C." + ], + "tags": [ + "transitive" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -3.1261911392211914, + "mean_logprob": -0.4465987341744559, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -10.728402137756348, + "mean_logprob": -1.7880670229593914, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -14.119539260864258, + "mean_logprob": -3.5298848152160645, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -5.211345195770264, + "mean_logprob": -0.7444778851100377, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.29787915093558176, + "context_truncated": false, + "elapsed_seconds": 0.027715070000340347 + }, + { + "id": "logical_reasoning-005", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Either the coin is in the cup or in the box. It is not in the cup. Thus it is", + "continuations": [ + " in the box.", + " in both places.", + " nowhere.", + " still in the cup." + ], + "tags": [ + "disjunction" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -4.2625555992126465, + "mean_logprob": -1.0656388998031616, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -11.732442855834961, + "mean_logprob": -2.9331107139587402, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -21.61458969116211, + "mean_logprob": -7.20486323038737, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -8.50075912475586, + "mean_logprob": -1.7001518249511718, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.6345129251480102, + "context_truncated": false, + "elapsed_seconds": 0.027197057999728713 + }, + { + "id": "logical_reasoning-006", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Every metal key conducts electricity. This object is a metal key, so it", + "continuations": [ + " cannot be metal.", + " conducts electricity.", + " must be wooden.", + " proves all objects conduct." + ], + "tags": [ + "deduction" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -16.674589157104492, + "mean_logprob": -4.168647289276123, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -4.755519866943359, + "mean_logprob": -1.1888799667358398, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -14.105069160461426, + "mean_logprob": -3.5262672901153564, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -37.62774658203125, + "mean_logprob": -7.52554931640625, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 2.3373873233795166, + "context_truncated": false, + "elapsed_seconds": 0.027393565998863778 + }, + { + "id": "logical_reasoning-007", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "If today is Monday, tomorrow is Tuesday. Today is Monday. Tomorrow is", + "continuations": [ + " Sunday.", + " Wednesday.", + " Tuesday.", + " Monday." + ], + "tags": [ + "conditional" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -4.646546363830566, + "mean_logprob": -2.323273181915283, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -2.154924154281616, + "mean_logprob": -1.077462077140808, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -0.5209131240844727, + "mean_logprob": -0.26045656204223633, + "total_sequence_tokens": 18 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -4.987026214599609, + "mean_logprob": -2.4935131072998047, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.8170055150985718, + "context_truncated": false, + "elapsed_seconds": 0.03134048799984157 + }, + { + "id": "logical_reasoning-008", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "No red token is large. Token P is red. Token P is", + "continuations": [ + " definitely large.", + " both red and large.", + " not red.", + " not large." + ], + "tags": [ + "deduction" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -14.585792541503906, + "mean_logprob": -4.861930847167969, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 5, + "sum_logprob": -10.64543342590332, + "mean_logprob": -2.1290866851806642, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -2.6387014389038086, + "mean_logprob": -0.8795671463012695, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -6.280209541320801, + "mean_logprob": -2.093403180440267, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 1.2138360341389975, + "context_truncated": false, + "elapsed_seconds": 0.027026356001442764 + }, + { + "id": "logical_reasoning-009", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Only one of Door A and Door B is open. Door A is closed, so", + "continuations": [ + " Door B is open.", + " both are closed.", + " both are open.", + " Door B must be closed." + ], + "tags": [ + "exclusive_or" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -2.959444761276245, + "mean_logprob": -0.49324079354604083, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -12.22856330871582, + "mean_logprob": -3.057140827178955, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -10.97695541381836, + "mean_logprob": -2.74423885345459, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -4.827787399291992, + "mean_logprob": -0.6896839141845703, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.19644312063852948, + "context_truncated": false, + "elapsed_seconds": 0.02870630799952778 + }, + { + "id": "logical_reasoning-010", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "All cedar trees are plants. No plant is a machine. Therefore, no cedar tree is", + "continuations": [ + " a plant.", + " a machine.", + " a tree.", + " alive." + ], + "tags": [ + "syllogism" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -7.419729232788086, + "mean_logprob": -2.473243077596029, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -0.6436737775802612, + "mean_logprob": -0.21455792586008707, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -11.101336479187012, + "mean_logprob": -3.7004454930623374, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -7.84523868560791, + "mean_logprob": -3.922619342803955, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 2.258685151735942, + "context_truncated": false, + "elapsed_seconds": 0.026393262000055984 + }, + { + "id": "logical_reasoning-011", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "If it rains, the path gets wet. The path is not wet. Therefore, it", + "continuations": [ + " certainly snowed.", + " must be night.", + " did not rain.", + " is still raining." + ], + "tags": [ + "modus_tollens" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -18.82676887512207, + "mean_logprob": -4.706692218780518, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -10.50233268737793, + "mean_logprob": -2.6255831718444824, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -6.35506010055542, + "mean_logprob": -1.588765025138855, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -9.369444847106934, + "mean_logprob": -1.8738889694213867, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 0.28512394428253174, + "context_truncated": false, + "elapsed_seconds": 0.02827742499903252 + }, + { + "id": "logical_reasoning-012", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "L is before M, and M is before N. In this ordering, N is", + "continuations": [ + " first.", + " before L.", + " between L and M.", + " after L." + ], + "tags": [ + "order" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -5.804980278015137, + "mean_logprob": -2.9024901390075684, + "total_sequence_tokens": 19 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -9.521125793457031, + "mean_logprob": -3.1737085978190103, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -10.34437084197998, + "mean_logprob": -2.068874168395996, + "total_sequence_tokens": 22 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -9.500274658203125, + "mean_logprob": -3.1667582194010415, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.8336159706115724, + "context_truncated": false, + "elapsed_seconds": 0.028135656999438652 + }, + { + "id": "logical_reasoning-013", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "Some poets are teachers. Every teacher reads. Therefore, some poets", + "continuations": [ + " read.", + " never read.", + " are not teachers.", + " are all teachers." + ], + "tags": [ + "syllogism" + ], + "prediction": 2, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -5.698014736175537, + "mean_logprob": -2.8490073680877686, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -7.821164131164551, + "mean_logprob": -2.6070547103881836, + "total_sequence_tokens": 18 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -6.846439361572266, + "mean_logprob": -1.7116098403930664, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -8.271710395812988, + "mean_logprob": -2.067927598953247, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.35631775856018066, + "context_truncated": false, + "elapsed_seconds": 0.03454536499884853 + }, + { + "id": "logical_reasoning-014", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "Exactly one card is marked. Card X is not marked and Card Y is marked. Card Z is therefore", + "continuations": [ + " also marked.", + " not marked.", + " the same as Y.", + " impossible to classify." + ], + "tags": [ + "constraint" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -5.6638665199279785, + "mean_logprob": -1.8879555066426594, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -1.8135178089141846, + "mean_logprob": -0.6045059363047282, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -19.507362365722656, + "mean_logprob": -3.901472473144531, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -23.16461181640625, + "mean_logprob": -4.63292236328125, + "total_sequence_tokens": 26 + } + ], + "best_vs_second_margin": 1.2834495703379312, + "context_truncated": false, + "elapsed_seconds": 0.027377290000003995 + }, + { + "id": "logical_reasoning-015", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 1.35, + "item_elo": 950, + "context": "If a number is divisible by four, it is even. Twenty is divisible by four, so twenty is", + "continuations": [ + " odd.", + " prime.", + " even.", + " negative." + ], + "tags": [ + "deduction" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -6.139820098876953, + "mean_logprob": -3.0699100494384766, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -10.674606323242188, + "mean_logprob": -5.337303161621094, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -1.3192028999328613, + "mean_logprob": -0.6596014499664307, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -15.370566368103027, + "mean_logprob": -7.685283184051514, + "total_sequence_tokens": 24 + } + ], + "best_vs_second_margin": 2.410308599472046, + "context_truncated": false, + "elapsed_seconds": 0.025267508999604615 + }, + { + "id": "logical_reasoning-016", + "category": "logical_reasoning", + "difficulty": "easy", + "category_weight": 1.35, + "difficulty_weight": 1.0, + "point_weight": 1.35, + "earned_points": 0.0, + "item_elo": 950, + "context": "No blue item is fragile. Vase V is fragile. Therefore, Vase V is", + "continuations": [ + " definitely blue.", + " both blue and fragile.", + " not fragile.", + " not blue." + ], + "tags": [ + "contraposition" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -13.683470726013184, + "mean_logprob": -4.5611569086710615, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -9.270719528198242, + "mean_logprob": -1.5451199213663738, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -4.348690032958984, + "mean_logprob": -1.087172508239746, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -7.262202262878418, + "mean_logprob": -2.420734087626139, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.4579474131266277, + "context_truncated": false, + "elapsed_seconds": 0.02956127200013725 + }, + { + "id": "logical_reasoning-017", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "All glims are nords. Some nords are pelts. From these facts alone, it follows that", + "continuations": [ + " no conclusion links glims to pelts.", + " all glims are pelts.", + " some glims are pelts.", + " no nords are glims." + ], + "tags": [ + "quantifiers" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -37.376155853271484, + "mean_logprob": -4.152906205919054, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -13.048042297363281, + "mean_logprob": -1.8640060424804688, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -16.346662521362305, + "mean_logprob": -2.335237503051758, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -17.042264938354492, + "mean_logprob": -2.4346092769077847, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.47123146057128906, + "context_truncated": false, + "elapsed_seconds": 0.0377511910010071 + }, + { + "id": "logical_reasoning-018", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "If R then S. If S then T. R is true. Therefore,", + "continuations": [ + " T is false.", + " T is true.", + " S is false.", + " R is false." + ], + "tags": [ + "conditional_chain" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -3.9769163131713867, + "mean_logprob": -0.9942290782928467, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -3.0385985374450684, + "mean_logprob": -0.7596496343612671, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -5.500736236572266, + "mean_logprob": -1.3751840591430664, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -6.0516676902771, + "mean_logprob": -1.512916922569275, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.2345794439315796, + "context_truncated": false, + "elapsed_seconds": 0.035311272000399185 + }, + { + "id": "logical_reasoning-019", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Exactly two of A, B, and C are true. A is true and B is false. Therefore, C is", + "continuations": [ + " false.", + " unknown.", + " true.", + " both true and false." + ], + "tags": [ + "constraint" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -0.36741650104522705, + "mean_logprob": -0.18370825052261353, + "total_sequence_tokens": 26 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -10.698878288269043, + "mean_logprob": -5.3494391441345215, + "total_sequence_tokens": 26 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -1.841135859489441, + "mean_logprob": -0.9205679297447205, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -7.60679292678833, + "mean_logprob": -1.521358585357666, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.7368596792221069, + "context_truncated": false, + "elapsed_seconds": 0.03049728999940271 + }, + { + "id": "logical_reasoning-020", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "Kai sits somewhere left of Lio. Mina sits right of Lio. Which statement must be true?", + "continuations": [ + " Mina sits left of Kai.", + " Kai sits beside Mina.", + " Lio sits left of Kai.", + " Kai sits left of Mina." + ], + "tags": [ + "spatial" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -20.981042861938477, + "mean_logprob": -2.3312269846598306, + "total_sequence_tokens": 34 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -22.95328140258789, + "mean_logprob": -2.5503646002875433, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -21.372692108154297, + "mean_logprob": -2.3747435675726996, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -21.874128341674805, + "mean_logprob": -2.4304587046305337, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.04351658291286897, + "context_truncated": false, + "elapsed_seconds": 0.03377929599992058 + }, + { + "id": "logical_reasoning-021", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Every zor is a vek, and no vek is a lum. Therefore,", + "continuations": [ + " no zor is a lum.", + " every lum is a zor.", + " some zors are lums.", + " no zor is a vek." + ], + "tags": [ + "syllogism" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -9.927262306213379, + "mean_logprob": -1.2409077882766724, + "total_sequence_tokens": 27 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -12.893324851989746, + "mean_logprob": -1.6116656064987183, + "total_sequence_tokens": 27 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -23.006277084350586, + "mean_logprob": -3.2866110120500838, + "total_sequence_tokens": 26 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -12.96645736694336, + "mean_logprob": -1.62080717086792, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.3707578182220459, + "context_truncated": false, + "elapsed_seconds": 0.029061327000817982 + }, + { + "id": "logical_reasoning-022", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "The alarm rings if and only if the sensor is active. The alarm is silent. Thus the sensor is", + "continuations": [ + " active.", + " inactive.", + " both states.", + " unrelated to the alarm." + ], + "tags": [ + "biconditional" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -3.9675488471984863, + "mean_logprob": -1.9837744235992432, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -4.27786111831665, + "mean_logprob": -2.138930559158325, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -24.123218536376953, + "mean_logprob": -8.041072845458984, + "total_sequence_tokens": 24 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -11.43774127960205, + "mean_logprob": -1.9062902132670085, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.07748421033223463, + "context_truncated": false, + "elapsed_seconds": 0.026720802999989246 + }, + { + "id": "logical_reasoning-023", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "At least one of P or Q is false. P is true. It follows that Q is", + "continuations": [ + " true.", + " undetermined.", + " false.", + " equal to P." + ], + "tags": [ + "propositional" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -1.5732948780059814, + "mean_logprob": -0.7866474390029907, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -17.106487274169922, + "mean_logprob": -4.2766218185424805, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -0.8197979927062988, + "mean_logprob": -0.4098989963531494, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -10.352224349975586, + "mean_logprob": -2.5880560874938965, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 0.3767484426498413, + "context_truncated": false, + "elapsed_seconds": 0.025991026999690803 + }, + { + "id": "logical_reasoning-024", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Three runners finish with no ties. Ada is not first. Bo finishes after Cy. The only possible first-place runner is", + "continuations": [ + " Ada.", + " Bo.", + " either Ada or Bo.", + " Cy." + ], + "tags": [ + "constraint" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -7.815156936645508, + "mean_logprob": -3.907578468322754, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -7.207143783569336, + "mean_logprob": -3.603571891784668, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -18.599592208862305, + "mean_logprob": -3.719918441772461, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -6.466699123382568, + "mean_logprob": -3.233349561691284, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.3702223300933838, + "context_truncated": false, + "elapsed_seconds": 0.026135096999496454 + }, + { + "id": "logical_reasoning-025", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "If the report is approved, funding continues. Funding did not continue. Therefore, the report was", + "continuations": [ + " not approved.", + " approved late.", + " necessarily reviewed.", + " unrelated to funding." + ], + "tags": [ + "modus_tollens" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -5.09077262878418, + "mean_logprob": -1.6969242095947266, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -17.806264877319336, + "mean_logprob": -5.935421625773112, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -21.076892852783203, + "mean_logprob": -7.025630950927734, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -18.95391273498535, + "mean_logprob": -3.7907825469970704, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 2.093858337402344, + "context_truncated": false, + "elapsed_seconds": 0.06509407600060513 + }, + { + "id": "logical_reasoning-026", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Some artists are engineers, and no engineer is careless. Therefore, some artists are", + "continuations": [ + " careless.", + " not careless.", + " not engineers.", + " all engineers." + ], + "tags": [ + "syllogism" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -6.2993855476379395, + "mean_logprob": -2.09979518254598, + "total_sequence_tokens": 20 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -7.499154090881348, + "mean_logprob": -1.874788522720337, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -5.7073564529418945, + "mean_logprob": -1.9024521509806316, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -9.565094947814941, + "mean_logprob": -3.1883649826049805, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.02766362826029467, + "context_truncated": false, + "elapsed_seconds": 0.06115498600047431 + }, + { + "id": "logical_reasoning-027", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "A statement and its negation cannot both be true. The statement is true, so its negation is", + "continuations": [ + " true.", + " unknown.", + " false.", + " equivalent." + ], + "tags": [ + "negation" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -4.395155906677246, + "mean_logprob": -2.197577953338623, + "total_sequence_tokens": 21 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -14.564260482788086, + "mean_logprob": -7.282130241394043, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -0.576337456703186, + "mean_logprob": -0.288168728351593, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -14.92807674407959, + "mean_logprob": -7.464038372039795, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 1.90940922498703, + "context_truncated": false, + "elapsed_seconds": 0.0335494060000201 + }, + { + "id": "logical_reasoning-028", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "If X is selected, Y must be selected. If Y is selected, Z cannot be selected. X is selected, so Z", + "continuations": [ + " must be selected.", + " may equal X.", + " makes Y unselected.", + " cannot be selected." + ], + "tags": [ + "constraint" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -1.6457048654556274, + "mean_logprob": -0.41142621636390686, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -17.842538833618164, + "mean_logprob": -4.460634708404541, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -15.62555980682373, + "mean_logprob": -3.125111961364746, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -4.505889415740967, + "mean_logprob": -1.1264723539352417, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.7150461375713348, + "context_truncated": false, + "elapsed_seconds": 0.03742306400090456 + }, + { + "id": "logical_reasoning-029", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 0.0, + "item_elo": 1050, + "context": "No historians are timeless beings. Some writers are historians. Therefore,", + "continuations": [ + " some writers are not timeless beings.", + " all writers are timeless.", + " no writers are historians.", + " all historians are writers." + ], + "tags": [ + "syllogism" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -24.91695785522461, + "mean_logprob": -3.114619731903076, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -15.53573989868164, + "mean_logprob": -2.589289983113607, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -12.175522804260254, + "mean_logprob": -2.435104560852051, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -11.63157844543457, + "mean_logprob": -2.326315689086914, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.10878887176513707, + "context_truncated": false, + "elapsed_seconds": 0.028489014999649953 + }, + { + "id": "logical_reasoning-030", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Either J or K, but not both, is open. If J is open, L is closed. J is open. Hence", + "continuations": [ + " K and L are open.", + " K and L are closed.", + " K is open and L closed.", + " K is closed and L open." + ], + "tags": [ + "propositional" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -11.102985382080078, + "mean_logprob": -1.850497563680013, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -10.669707298278809, + "mean_logprob": -1.7782845497131348, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -13.543540954589844, + "mean_logprob": -1.9347915649414062, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -15.956045150756836, + "mean_logprob": -2.279435021536691, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 0.07221301396687818, + "context_truncated": false, + "elapsed_seconds": 0.039278928999920026 + }, + { + "id": "logical_reasoning-031", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Rhea is before Sami. Taro is after Sami. Ula is before Rhea. The second person in order is", + "continuations": [ + " Ula.", + " Sami.", + " Rhea.", + " Taro." + ], + "tags": [ + "order" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -3.3497650623321533, + "mean_logprob": -1.1165883541107178, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -2.953437566757202, + "mean_logprob": -0.9844791889190674, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -2.262697458267212, + "mean_logprob": -0.7542324860890707, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -2.8338801860809326, + "mean_logprob": -0.9446267286936442, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.1903942426045735, + "context_truncated": false, + "elapsed_seconds": 0.02460370500011777 + }, + { + "id": "logical_reasoning-032", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "All valid tickets have a stamp. This ticket has no stamp. It is therefore", + "continuations": [ + " valid.", + " stamped invisibly.", + " both valid and invalid.", + " not valid." + ], + "tags": [ + "contraposition" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -2.1518657207489014, + "mean_logprob": -1.0759328603744507, + "total_sequence_tokens": 18 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -25.669761657714844, + "mean_logprob": -4.278293609619141, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -8.00202751159668, + "mean_logprob": -1.600405502319336, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -2.469339370727539, + "mean_logprob": -0.8231131235758463, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.25281973679860437, + "context_truncated": false, + "elapsed_seconds": 0.02711293900028977 + }, + { + "id": "logical_reasoning-033", + "category": "logical_reasoning", + "difficulty": "medium", + "category_weight": 1.35, + "difficulty_weight": 1.5, + "point_weight": 2.0250000000000004, + "earned_points": 2.0250000000000004, + "item_elo": 1050, + "context": "Exactly one of the claims \"A is true\" and \"B is false\" is true. A is false. Therefore, B is", + "continuations": [ + " false.", + " true.", + " both.", + " undetermined." + ], + "tags": [ + "propositional" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -0.25394731760025024, + "mean_logprob": -0.12697365880012512, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -3.023630380630493, + "mean_logprob": -1.5118151903152466, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 2, + "sum_logprob": -17.4687557220459, + "mean_logprob": -8.73437786102295, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -17.809072494506836, + "mean_logprob": -4.452268123626709, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 1.3848415315151215, + "context_truncated": false, + "elapsed_seconds": 0.03756342099950416 + }, + { + "id": "logical_reasoning-034", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Every quill is a renn. No renn is a soth. Some varks are quills. It follows that some varks are", + "continuations": [ + " soths.", + " not soths.", + " not quills.", + " all renns." + ], + "tags": [ + "syllogism" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 3, + "sum_logprob": -5.736642837524414, + "mean_logprob": -1.9122142791748047, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -8.529621124267578, + "mean_logprob": -2.1324052810668945, + "total_sequence_tokens": 34 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -3.5806849002838135, + "mean_logprob": -0.8951712250709534, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -20.263328552246094, + "mean_logprob": -5.065832138061523, + "total_sequence_tokens": 34 + } + ], + "best_vs_second_margin": 1.0170430541038513, + "context_truncated": false, + "elapsed_seconds": 0.037289870999302366 + }, + { + "id": "logical_reasoning-035", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Exactly two of P, Q, R, and S are true. P and Q have opposite truth values. R is true and S is false. Therefore,", + "continuations": [ + " P and Q are both true.", + " P and Q are both false.", + " exactly one of P and Q is true.", + " neither P nor Q matters." + ], + "tags": [ + "constraint" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -6.264081954956055, + "mean_logprob": -0.8948688507080078, + "total_sequence_tokens": 39 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -6.364994525909424, + "mean_logprob": -0.9092849322727748, + "total_sequence_tokens": 39 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -19.117538452148438, + "mean_logprob": -2.124170939127604, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -13.39778995513916, + "mean_logprob": -2.2329649925231934, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 0.014416081564767036, + "context_truncated": false, + "elapsed_seconds": 0.03755214200100454 + }, + { + "id": "logical_reasoning-036", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "If A then B. If not C then not B. A is true. Which conclusion must follow?", + "continuations": [ + " C is false.", + " B is false.", + " A is false.", + " C is true." + ], + "tags": [ + "conditional_chain" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -17.453317642211914, + "mean_logprob": -4.3633294105529785, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -17.41801643371582, + "mean_logprob": -4.354504108428955, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -16.327856063842773, + "mean_logprob": -4.081964015960693, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -17.262537002563477, + "mean_logprob": -4.315634250640869, + "total_sequence_tokens": 25 + } + ], + "best_vs_second_margin": 0.23367023468017578, + "context_truncated": false, + "elapsed_seconds": 0.027254909999101073 + }, + { + "id": "logical_reasoning-037", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "Five seats are numbered 1 through 5. Iva sits immediately left of Jae. Kian sits in seat 5. Iva sits in seat 2. Jae sits in", + "continuations": [ + " seat 3.", + " seat 1.", + " seat 4.", + " seat 5." + ], + "tags": [ + "constraint" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -1.0388476848602295, + "mean_logprob": -0.2597119212150574, + "total_sequence_tokens": 48 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -1.3396497964859009, + "mean_logprob": -0.3349124491214752, + "total_sequence_tokens": 48 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -1.67790687084198, + "mean_logprob": -0.419476717710495, + "total_sequence_tokens": 48 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -5.073331356048584, + "mean_logprob": -1.268332839012146, + "total_sequence_tokens": 48 + } + ], + "best_vs_second_margin": 0.07520052790641785, + "context_truncated": false, + "elapsed_seconds": 0.03574124800070422 + }, + { + "id": "logical_reasoning-038", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "No fal is a mer. Every tov is a fal. Some riks are mers. Which statement must be true?", + "continuations": [ + " Some riks are tovs.", + " No tov is a mer.", + " Every rik is a fal.", + " Some mers are fals." + ], + "tags": [ + "quantifiers" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -20.39111328125, + "mean_logprob": -2.9130161830357144, + "total_sequence_tokens": 35 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -18.03400230407715, + "mean_logprob": -2.5762860434395924, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -15.599857330322266, + "mean_logprob": -1.9499821662902832, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -17.913246154785156, + "mean_logprob": -2.559035164969308, + "total_sequence_tokens": 35 + } + ], + "best_vs_second_margin": 0.609052998679025, + "context_truncated": false, + "elapsed_seconds": 0.0381310630000371 + }, + { + "id": "logical_reasoning-039", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "A safe opens exactly when both keys are turned. The safe did not open, and the first key was turned. Therefore, the second key was", + "continuations": [ + " turned.", + " broken necessarily.", + " not turned.", + " unrelated." + ], + "tags": [ + "conjunction" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -3.114652633666992, + "mean_logprob": -1.557326316833496, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -24.938669204711914, + "mean_logprob": -8.31288973490397, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -6.651601314544678, + "mean_logprob": -2.217200438181559, + "total_sequence_tokens": 32 + }, + { + "index": 3, + "tokens": 3, + "sum_logprob": -13.41206169128418, + "mean_logprob": -4.47068723042806, + "total_sequence_tokens": 32 + } + ], + "best_vs_second_margin": 0.659874121348063, + "context_truncated": false, + "elapsed_seconds": 0.025557093000315945 + }, + { + "id": "logical_reasoning-040", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "A committee selects exactly three of J, K, L, and M. If J is selected, K is not. L and M are selected. Therefore, the third member must be", + "continuations": [ + " J.", + " L again.", + " M again.", + " K." + ], + "tags": [ + "constraint" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -6.013801574707031, + "mean_logprob": -3.0069007873535156, + "total_sequence_tokens": 40 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -14.643722534179688, + "mean_logprob": -4.8812408447265625, + "total_sequence_tokens": 41 + }, + { + "index": 2, + "tokens": 3, + "sum_logprob": -15.19642448425293, + "mean_logprob": -5.06547482808431, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -4.991424560546875, + "mean_logprob": -2.4957122802734375, + "total_sequence_tokens": 40 + } + ], + "best_vs_second_margin": 0.5111885070800781, + "context_truncated": false, + "elapsed_seconds": 0.03555398400021659 + }, + { + "id": "logical_reasoning-041", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Some nels are paks. All paks are wims. No wim is a dor. Therefore,", + "continuations": [ + " some nels are not dors.", + " all nels are wims.", + " some dors are paks.", + " no nels are paks." + ], + "tags": [ + "syllogism" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -17.275667190551758, + "mean_logprob": -2.1594583988189697, + "total_sequence_tokens": 32 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -7.622884750366211, + "mean_logprob": -1.0889835357666016, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -15.269927978515625, + "mean_logprob": -2.1814182826450894, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -11.179550170898438, + "mean_logprob": -1.597078595842634, + "total_sequence_tokens": 31 + } + ], + "best_vs_second_margin": 0.5080950600760323, + "context_truncated": false, + "elapsed_seconds": 0.03802208500019333 + }, + { + "id": "logical_reasoning-042", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Four tasks W, X, Y, Z are scheduled once each. W precedes X. Y immediately precedes Z. X is first. These conditions are", + "continuations": [ + " satisfied by X, W, Y, Z.", + " impossible to satisfy.", + " satisfied by W, X, Y, Z.", + " satisfied by Y, Z, W, X." + ], + "tags": [ + "consistency" + ], + "prediction": 2, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -18.201345443725586, + "mean_logprob": -1.8201345443725585, + "total_sequence_tokens": 42 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -10.938289642333984, + "mean_logprob": -2.734572410583496, + "total_sequence_tokens": 36 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -15.769869804382324, + "mean_logprob": -1.5769869804382324, + "total_sequence_tokens": 42 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -23.666305541992188, + "mean_logprob": -2.3666305541992188, + "total_sequence_tokens": 42 + } + ], + "best_vs_second_margin": 0.24314756393432613, + "context_truncated": false, + "elapsed_seconds": 0.04649977599910926 + }, + { + "id": "logical_reasoning-043", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "If either M or N is true, P is true. P is false. Therefore,", + "continuations": [ + " M is true.", + " N is true.", + " both M and N are false.", + " exactly one is false." + ], + "tags": [ + "contraposition" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -9.543510437011719, + "mean_logprob": -2.3858776092529297, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -9.155370712280273, + "mean_logprob": -2.2888426780700684, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -6.482165336608887, + "mean_logprob": -0.9260236195155552, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -15.393881797790527, + "mean_logprob": -3.0787763595581055, + "total_sequence_tokens": 23 + } + ], + "best_vs_second_margin": 1.3628190585545132, + "context_truncated": false, + "elapsed_seconds": 0.026972736000971054 + }, + { + "id": "logical_reasoning-044", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "All A-things are B-things. Some B-things are C-things. No C-thing is a D-thing. Which claim is guaranteed?", + "continuations": [ + " Some A-things are C-things.", + " No A-thing is a D-thing.", + " All B-things are A-things.", + " Some B-things are not D-things." + ], + "tags": [ + "quantifiers" + ], + "prediction": 2, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -12.665159225463867, + "mean_logprob": -1.1513781114058061, + "total_sequence_tokens": 48 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -14.303797721862793, + "mean_logprob": -1.4303797721862792, + "total_sequence_tokens": 47 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -10.99329662322998, + "mean_logprob": -0.9993906021118164, + "total_sequence_tokens": 48 + }, + { + "index": 3, + "tokens": 12, + "sum_logprob": -14.286218643188477, + "mean_logprob": -1.1905182202657063, + "total_sequence_tokens": 49 + } + ], + "best_vs_second_margin": 0.15198750929398974, + "context_truncated": false, + "elapsed_seconds": 0.04841408700121974 + }, + { + "id": "logical_reasoning-045", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "A code is valid if exactly one of conditions X and Y holds. X holds and the code is valid. Therefore, Y", + "continuations": [ + " does not hold.", + " also holds.", + " is equivalent to X.", + " cannot be discussed." + ], + "tags": [ + "exclusive_or" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -3.2396998405456543, + "mean_logprob": -0.8099249601364136, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -4.657301425933838, + "mean_logprob": -1.5524338086446126, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -10.75133991241455, + "mean_logprob": -2.15026798248291, + "total_sequence_tokens": 30 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -20.620361328125, + "mean_logprob": -5.15509033203125, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.742508848508199, + "context_truncated": false, + "elapsed_seconds": 0.031652417999794125 + }, + { + "id": "logical_reasoning-046", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "R is earlier than S. T is later than U. S is earlier than U. Which ordering must hold?", + "continuations": [ + " T is earlier than R.", + " R is earlier than T.", + " U is earlier than S.", + " S is later than T." + ], + "tags": [ + "transitive" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -20.732337951660156, + "mean_logprob": -3.455389658610026, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -14.730389595031738, + "mean_logprob": -2.4550649325052896, + "total_sequence_tokens": 29 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -20.681095123291016, + "mean_logprob": -3.4468491872151694, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -16.98881721496582, + "mean_logprob": -2.8314695358276367, + "total_sequence_tokens": 29 + } + ], + "best_vs_second_margin": 0.37640460332234715, + "context_truncated": false, + "elapsed_seconds": 0.05448604800039902 + }, + { + "id": "logical_reasoning-047", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "If the engine starts, either fuel or battery power is available. The engine starts, and no fuel is available. Thus", + "continuations": [ + " no power is available.", + " the engine did not start.", + " battery power is available.", + " fuel is available." + ], + "tags": [ + "disjunction" + ], + "prediction": 0, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 5, + "sum_logprob": -21.319812774658203, + "mean_logprob": -4.2639625549316404, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -25.933473587036133, + "mean_logprob": -4.3222455978393555, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -23.158662796020508, + "mean_logprob": -4.631732559204101, + "total_sequence_tokens": 29 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -22.04351806640625, + "mean_logprob": -5.5108795166015625, + "total_sequence_tokens": 28 + } + ], + "best_vs_second_margin": 0.05828304290771502, + "context_truncated": false, + "elapsed_seconds": 0.03828422100013995 + }, + { + "id": "logical_reasoning-048", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "Exactly one person among A, B, and C lies. A says B tells the truth. B says C tells the truth. C says A lies. The liar is", + "continuations": [ + " A.", + " B.", + " impossible to determine.", + " C." + ], + "tags": [ + "truth_tellers" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -4.278550624847412, + "mean_logprob": -2.139275312423706, + "total_sequence_tokens": 37 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -8.535700798034668, + "mean_logprob": -4.267850399017334, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -15.505983352661133, + "mean_logprob": -3.876495838165283, + "total_sequence_tokens": 39 + }, + { + "index": 3, + "tokens": 2, + "sum_logprob": -11.253164291381836, + "mean_logprob": -5.626582145690918, + "total_sequence_tokens": 37 + } + ], + "best_vs_second_margin": 1.7372205257415771, + "context_truncated": false, + "elapsed_seconds": 0.03660139399835316 + }, + { + "id": "logical_reasoning-049", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 0.0, + "item_elo": 1200, + "context": "No glens are haps. Some jors are glens. Every jor is a mip. It follows that some mips are", + "continuations": [ + " not haps.", + " haps.", + " not jors.", + " all glens." + ], + "tags": [ + "syllogism" + ], + "prediction": 1, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -8.938095092773438, + "mean_logprob": -2.2345237731933594, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 3, + "sum_logprob": -2.8486106395721436, + "mean_logprob": -0.9495368798573812, + "total_sequence_tokens": 32 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -8.137388229370117, + "mean_logprob": -2.0343470573425293, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -5.661764144897461, + "mean_logprob": -1.4154410362243652, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.465904156366984, + "context_truncated": false, + "elapsed_seconds": 0.03652822299955005 + }, + { + "id": "logical_reasoning-050", + "category": "logical_reasoning", + "difficulty": "hard", + "category_weight": 1.35, + "difficulty_weight": 2.25, + "point_weight": 3.0375, + "earned_points": 3.0375, + "item_elo": 1200, + "context": "A proposition is true only if Q is true, and Q is true only if R is false. The proposition is true. Therefore, R is", + "continuations": [ + " true.", + " false.", + " both true and false.", + " independent of Q." + ], + "tags": [ + "conditional_chain" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 2, + "sum_logprob": -2.581470012664795, + "mean_logprob": -1.2907350063323975, + "total_sequence_tokens": 31 + }, + { + "index": 1, + "tokens": 2, + "sum_logprob": -0.4033554494380951, + "mean_logprob": -0.20167772471904755, + "total_sequence_tokens": 31 + }, + { + "index": 2, + "tokens": 5, + "sum_logprob": -5.504075527191162, + "mean_logprob": -1.1008151054382325, + "total_sequence_tokens": 34 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -9.2425537109375, + "mean_logprob": -2.310638427734375, + "total_sequence_tokens": 33 + } + ], + "best_vs_second_margin": 0.8991373807191849, + "context_truncated": false, + "elapsed_seconds": 0.03524178599946026 + }, + { + "id": "code_completion-001", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def square(x):\n", + "continuations": [ + " return x * x\n", + " return x + x\n", + " print(x)\n", + " return 2\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -5.934330463409424, + "mean_logprob": -0.8477614947727748, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -14.469461441040039, + "mean_logprob": -2.067065920148577, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -9.781411170959473, + "mean_logprob": -1.630235195159912, + "total_sequence_tokens": 12 + }, + { + "index": 3, + "tokens": 5, + "sum_logprob": -18.043697357177734, + "mean_logprob": -3.608739471435547, + "total_sequence_tokens": 11 + } + ], + "best_vs_second_margin": 0.7824737003871373, + "context_truncated": false, + "elapsed_seconds": 0.026375286000984488 + }, + { + "id": "code_completion-002", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def is_empty(items):\n", + "continuations": [ + " return items[0]\n", + " return len(items) == 0\n", + " return len(items) + 1\n", + " items.clear()\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -14.702980995178223, + "mean_logprob": -2.1004258564540317, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -8.024712562561035, + "mean_logprob": -0.729519323869185, + "total_sequence_tokens": 19 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -16.210472106933594, + "mean_logprob": -1.4736792824485085, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -9.871957778930664, + "mean_logprob": -1.645326296488444, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.7441599585793235, + "context_truncated": false, + "elapsed_seconds": 0.0282552590015257 + }, + { + "id": "code_completion-003", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def greet(name):\n", + "continuations": [ + " return name - \"Hello\"\n", + " print = name\n", + " return \"Hello, \" + name\n", + " return len(\"Hello\")\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -24.306949615478516, + "mean_logprob": -2.4306949615478515, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -19.574186325073242, + "mean_logprob": -3.262364387512207, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -11.44089412689209, + "mean_logprob": -0.9534078439076742, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -27.81157684326172, + "mean_logprob": -3.9730824061802457, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 1.4772871176401772, + "context_truncated": false, + "elapsed_seconds": 0.03970513099920936 + }, + { + "id": "code_completion-004", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def first(items):\n", + "continuations": [ + " return items[-1]\n", + " return len(items)\n", + " return items\n", + " return items[0]\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -15.152753829956055, + "mean_logprob": -2.1646791185651506, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -14.308820724487305, + "mean_logprob": -2.0441172463553294, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 4, + "sum_logprob": -9.552918434143066, + "mean_logprob": -2.3882296085357666, + "total_sequence_tokens": 10 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -11.064523696899414, + "mean_logprob": -1.580646242414202, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.4634710039411274, + "context_truncated": false, + "elapsed_seconds": 0.027532070000233944 + }, + { + "id": "code_completion-005", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def add(a, b):\n", + "continuations": [ + " return a + b\n", + " return a - b\n", + " return a == b\n", + " return [a, b]\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -8.011276245117188, + "mean_logprob": -1.144468035016741, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -17.55638313293457, + "mean_logprob": -2.508054733276367, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -19.319067001342773, + "mean_logprob": -2.759866714477539, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -14.94140625, + "mean_logprob": -1.66015625, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.5156882149832589, + "context_truncated": false, + "elapsed_seconds": 0.026762985999084776 + }, + { + "id": "code_completion-006", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def last(items):\n", + "continuations": [ + " return items[0]\n", + " return items[-1]\n", + " return items[1:]\n", + " return -len(items)\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -12.57072639465332, + "mean_logprob": -1.7958180563790458, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -10.278488159179688, + "mean_logprob": -1.468355451311384, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -13.712234497070312, + "mean_logprob": -1.958890642438616, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -18.184860229492188, + "mean_logprob": -2.020540025499132, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.3274626050676619, + "context_truncated": false, + "elapsed_seconds": 0.03602598600082274 + }, + { + "id": "code_completion-007", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def is_even(n):\n", + "continuations": [ + " return n / 2\n", + " return n % 2 == 1\n", + " return n % 2 == 0\n", + " return n * 2\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -24.60840606689453, + "mean_logprob": -3.0760507583618164, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 12, + "sum_logprob": -19.844167709350586, + "mean_logprob": -1.653680642445882, + "total_sequence_tokens": 20 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -6.369754791259766, + "mean_logprob": -0.5308128992716471, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -24.57311248779297, + "mean_logprob": -3.071639060974121, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.1228677431742349, + "context_truncated": false, + "elapsed_seconds": 0.02552455099976214 + }, + { + "id": "code_completion-008", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def absolute_value(x):\n", + "continuations": [ + " return -abs(x)\n", + " return x * 0\n", + " return bool(x)\n", + " return abs(x)\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -11.221795082092285, + "mean_logprob": -1.2468661202324762, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -19.985536575317383, + "mean_logprob": -2.498192071914673, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -15.934894561767578, + "mean_logprob": -2.2764135088239397, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -6.373825550079346, + "mean_logprob": -0.9105465071541923, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.33631961307828395, + "context_truncated": false, + "elapsed_seconds": 0.027187195999431424 + }, + { + "id": "code_completion-009", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def double_all(numbers):\n", + "continuations": [ + " return [n * 2 for n in numbers]\n", + " return numbers * 0\n", + " return [2 for n in numbers]\n", + " return len(numbers) * 2\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 15, + "sum_logprob": -13.590858459472656, + "mean_logprob": -0.9060572306315104, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -21.125938415527344, + "mean_logprob": -2.640742301940918, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -25.994300842285156, + "mean_logprob": -2.3631182583895596, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -19.167556762695312, + "mean_logprob": -1.7425051602450283, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.8364479296135179, + "context_truncated": false, + "elapsed_seconds": 0.04008716399948753 + }, + { + "id": "code_completion-010", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def contains(items, value):\n", + "continuations": [ + " return items == value\n", + " return value in items\n", + " return items + value\n", + " return items.index\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -10.957561492919922, + "mean_logprob": -1.565365927559989, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -8.056070327758789, + "mean_logprob": -1.342678387959798, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -13.39298152923584, + "mean_logprob": -1.91328307560512, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -27.022537231445312, + "mean_logprob": -4.503756205240886, + "total_sequence_tokens": 14 + } + ], + "best_vs_second_margin": 0.22268753960019083, + "context_truncated": false, + "elapsed_seconds": 0.02570848799950909 + }, + { + "id": "code_completion-011", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def maximum(a, b):\n", + "continuations": [ + " return a + b\n", + " return a < b\n", + " return a if a > b else b\n", + " return None\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -12.686381340026855, + "mean_logprob": -1.812340191432408, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -12.447408676147461, + "mean_logprob": -1.7782012394496374, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -11.49631404876709, + "mean_logprob": -1.0451194589788264, + "total_sequence_tokens": 19 + }, + { + "index": 3, + "tokens": 4, + "sum_logprob": -16.26229476928711, + "mean_logprob": -4.065573692321777, + "total_sequence_tokens": 12 + } + ], + "best_vs_second_margin": 0.733081780470811, + "context_truncated": false, + "elapsed_seconds": 0.02868215599846735 + }, + { + "id": "code_completion-012", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def repeat(text, count):\n", + "continuations": [ + " return count / text\n", + " return len(text)\n", + " return text + count\n", + " return text * count\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -24.464187622070312, + "mean_logprob": -3.4948839460100447, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -15.841794967651367, + "mean_logprob": -2.263113566807338, + "total_sequence_tokens": 15 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -15.606399536132812, + "mean_logprob": -2.229485648018973, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -11.067680358886719, + "mean_logprob": -1.581097194126674, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.648388453892299, + "context_truncated": false, + "elapsed_seconds": 0.026035105998744257 + }, + { + "id": "code_completion-013", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def keys_list(mapping):\n", + "continuations": [ + " return list(mapping.keys())\n", + " return mapping.values\n", + " return [mapping]\n", + " return mapping.clear()\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -9.412752151489258, + "mean_logprob": -1.045861350165473, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -20.372913360595703, + "mean_logprob": -3.3954855600992837, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -17.14534568786621, + "mean_logprob": -2.4493350982666016, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -20.802125930786133, + "mean_logprob": -2.9717322758265903, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 1.4034737481011286, + "context_truncated": false, + "elapsed_seconds": 0.028490945000157808 + }, + { + "id": "code_completion-014", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def starts_with_a(text):\n", + "continuations": [ + " return text[-1] == \"a\"\n", + " return text.startswith(\"a\")\n", + " return \"a\" + text\n", + " return text.count\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -20.738313674926758, + "mean_logprob": -1.595254898071289, + "total_sequence_tokens": 23 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -11.440147399902344, + "mean_logprob": -1.040013399991122, + "total_sequence_tokens": 21 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -15.75004768371582, + "mean_logprob": -1.575004768371582, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 6, + "sum_logprob": -20.53750991821289, + "mean_logprob": -3.4229183197021484, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.5349913683804599, + "context_truncated": false, + "elapsed_seconds": 0.03216659999998228 + }, + { + "id": "code_completion-015", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def total(numbers):\n", + "continuations": [ + " return len(numbers)\n", + " return max(numbers)\n", + " return sum(numbers)\n", + " return numbers[0]\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -10.798761367797852, + "mean_logprob": -1.5426801953996931, + "total_sequence_tokens": 13 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -11.562196731567383, + "mean_logprob": -1.651742390223912, + "total_sequence_tokens": 13 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -6.728759288787842, + "mean_logprob": -0.9612513269696917, + "total_sequence_tokens": 13 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -17.91793441772461, + "mean_logprob": -2.559704916817801, + "total_sequence_tokens": 13 + } + ], + "best_vs_second_margin": 0.5814288684300014, + "context_truncated": false, + "elapsed_seconds": 0.02752010599942878 + }, + { + "id": "code_completion-016", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 0.0, + "item_elo": 950, + "context": "def clamp_zero(x):\n", + "continuations": [ + " return min(0, x)\n", + " return x == 0\n", + " return -x\n", + " return max(0, x)\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -17.8612060546875, + "mean_logprob": -1.9845784505208333, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -8.925451278686523, + "mean_logprob": -1.1156814098358154, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -12.237079620361328, + "mean_logprob": -2.039513270060221, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -11.028650283813477, + "mean_logprob": -1.2254055870903864, + "total_sequence_tokens": 17 + } + ], + "best_vs_second_margin": 0.10972417725457095, + "context_truncated": false, + "elapsed_seconds": 0.02768399599881377 + }, + { + "id": "code_completion-017", + "category": "code_completion", + "difficulty": "easy", + "category_weight": 1.4, + "difficulty_weight": 1.0, + "point_weight": 1.4, + "earned_points": 1.4, + "item_elo": 950, + "context": "def reversed_copy(items):\n", + "continuations": [ + " return items[::-1]\n", + " return items[::1]\n", + " return items.reverse\n", + " return sorted(items)\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -7.552888870239258, + "mean_logprob": -0.9441111087799072, + "total_sequence_tokens": 16 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -21.040647506713867, + "mean_logprob": -2.6300809383392334, + "total_sequence_tokens": 16 + }, + { + "index": 2, + "tokens": 6, + "sum_logprob": -18.220312118530273, + "mean_logprob": -3.0367186864217124, + "total_sequence_tokens": 14 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -12.42491626739502, + "mean_logprob": -1.7749880381992884, + "total_sequence_tokens": 15 + } + ], + "best_vs_second_margin": 0.8308769294193812, + "context_truncated": false, + "elapsed_seconds": 0.03015708699967945 + }, + { + "id": "code_completion-018", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def count_positive(numbers):\n", + "continuations": [ + " return sum(numbers)\n", + " return sum(1 for n in numbers if n > 0)\n", + " return [n > 0 for n in numbers]\n", + " return len(numbers) > 0\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -12.093025207519531, + "mean_logprob": -1.7275750296456474, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 17, + "sum_logprob": -15.259834289550781, + "mean_logprob": -0.897637311150046, + "total_sequence_tokens": 25 + }, + { + "index": 2, + "tokens": 15, + "sum_logprob": -15.651079177856445, + "mean_logprob": -1.043405278523763, + "total_sequence_tokens": 23 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -10.985816955566406, + "mean_logprob": -0.9987106323242188, + "total_sequence_tokens": 19 + } + ], + "best_vs_second_margin": 0.10107332117417278, + "context_truncated": false, + "elapsed_seconds": 0.03531654699872888 + }, + { + "id": "code_completion-019", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def unique_in_order(items):\n seen = set()\n", + "continuations": [ + " return set(items)\n", + " return sorted(items)\n", + " return [x for x in items if not (x in seen or seen.add(x))]\n", + " return list(reversed(items))\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -25.16358184814453, + "mean_logprob": -3.5947974068777904, + "total_sequence_tokens": 24 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -16.42459487915039, + "mean_logprob": -2.3463706970214844, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 24, + "sum_logprob": -38.50417709350586, + "mean_logprob": -1.6043407122294109, + "total_sequence_tokens": 41 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -23.19318389892578, + "mean_logprob": -2.319318389892578, + "total_sequence_tokens": 27 + } + ], + "best_vs_second_margin": 0.7149776776631673, + "context_truncated": false, + "elapsed_seconds": 0.03663187399979506 + }, + { + "id": "code_completion-020", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def safe_divide(a, b):\n if b == 0:\n", + "continuations": [ + " return a\n return b / a\n", + " b = 1\n return b\n", + " raise StopIteration\n return a + b\n", + " return None\n return a / b\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 3, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -14.361824989318848, + "mean_logprob": -1.3056204535744407, + "total_sequence_tokens": 30 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -31.49257469177246, + "mean_logprob": -2.8629613356156782, + "total_sequence_tokens": 30 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -27.718412399291992, + "mean_logprob": -2.3098676999409995, + "total_sequence_tokens": 31 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -19.33892250061035, + "mean_logprob": -1.75808386369185, + "total_sequence_tokens": 30 + } + ], + "best_vs_second_margin": 0.4524634101174094, + "context_truncated": false, + "elapsed_seconds": 0.04077255999982299 + }, + { + "id": "code_completion-021", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def word_lengths(words):\n", + "continuations": [ + " return {word: len(word) for word in words}\n", + " return {len(words): word for word in words}\n", + " return [words, len(words)]\n", + " return words.keys()\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 16, + "sum_logprob": -14.82179069519043, + "mean_logprob": -0.9263619184494019, + "total_sequence_tokens": 25 + }, + { + "index": 1, + "tokens": 15, + "sum_logprob": -34.53876495361328, + "mean_logprob": -2.3025843302408853, + "total_sequence_tokens": 24 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -27.680112838745117, + "mean_logprob": -2.5163738944313745, + "total_sequence_tokens": 20 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -24.124025344848633, + "mean_logprob": -3.4462893349783763, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 1.3762224117914834, + "context_truncated": false, + "elapsed_seconds": 0.03476182600024913 + }, + { + "id": "code_completion-022", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def flatten_once(groups):\n", + "continuations": [ + " return list(groups)\n", + " return [item for group in groups for item in group]\n", + " return [groups[0]]\n", + " return sum(len(g) for g in groups)\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -14.239151954650879, + "mean_logprob": -2.0341645649501254, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 15, + "sum_logprob": -24.19076156616211, + "mean_logprob": -1.6127174377441407, + "total_sequence_tokens": 23 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -18.42530059814453, + "mean_logprob": -2.047255622016059, + "total_sequence_tokens": 17 + }, + { + "index": 3, + "tokens": 14, + "sum_logprob": -21.162622451782227, + "mean_logprob": -1.5116158894130163, + "total_sequence_tokens": 22 + } + ], + "best_vs_second_margin": 0.10110154833112439, + "context_truncated": false, + "elapsed_seconds": 0.03249907099962002 + }, + { + "id": "code_completion-023", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def factorial(n):\n result = 1\n for value in range(2, n + 1):\n", + "continuations": [ + " result += value\n return n\n", + " result = value\n return value\n", + " result *= value\n return result\n", + " n -= value\n return n\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 10, + "sum_logprob": -27.79298210144043, + "mean_logprob": -2.779298210144043, + "total_sequence_tokens": 38 + }, + { + "index": 1, + "tokens": 10, + "sum_logprob": -37.84375762939453, + "mean_logprob": -3.784375762939453, + "total_sequence_tokens": 38 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -4.943729877471924, + "mean_logprob": -0.4943729877471924, + "total_sequence_tokens": 38 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -37.693763732910156, + "mean_logprob": -3.7693763732910157, + "total_sequence_tokens": 38 + } + ], + "best_vs_second_margin": 2.2849252223968506, + "context_truncated": false, + "elapsed_seconds": 0.03755112500039104 + }, + { + "id": "code_completion-024", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def first_index(items, target):\n for index, item in enumerate(items):\n if item == target:\n", + "continuations": [ + " continue\n return 0\n", + " return item\n return target\n", + " break\n return index\n", + " return index\n return -1\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -34.801002502441406, + "mean_logprob": -4.350125312805176, + "total_sequence_tokens": 37 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -26.48795509338379, + "mean_logprob": -3.3109943866729736, + "total_sequence_tokens": 37 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -13.000073432922363, + "mean_logprob": -1.8571533475603377, + "total_sequence_tokens": 36 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -8.2440185546875, + "mean_logprob": -0.82440185546875, + "total_sequence_tokens": 39 + } + ], + "best_vs_second_margin": 1.0327514920915877, + "context_truncated": false, + "elapsed_seconds": 0.0376353020001261 + }, + { + "id": "code_completion-025", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def merge_counts(left, right):\n result = dict(left)\n for key, value in right.items():\n", + "continuations": [ + " result[key] = result.get(key, 0) + value\n return result\n", + " result[value] = key\n return left\n", + " result.clear()\n return right\n", + " key += value\n return result.values()\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 24, + "sum_logprob": -8.848027229309082, + "mean_logprob": -0.36866780122121173, + "total_sequence_tokens": 54 + }, + { + "index": 1, + "tokens": 13, + "sum_logprob": -45.313819885253906, + "mean_logprob": -3.4856784527118387, + "total_sequence_tokens": 43 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -52.17894744873047, + "mean_logprob": -5.217894744873047, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 13, + "sum_logprob": -37.48686599731445, + "mean_logprob": -2.8836050767164965, + "total_sequence_tokens": 43 + } + ], + "best_vs_second_margin": 2.5149372754952846, + "context_truncated": false, + "elapsed_seconds": 0.048819237999850884 + }, + { + "id": "code_completion-026", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def chunks(items, size):\n", + "continuations": [ + " return items[:size]\n", + " return [items[i:i + size] for i in range(0, len(items), size)]\n", + " return [size] * len(items)\n", + " return items[::size][0]\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -12.145590782165527, + "mean_logprob": -1.735084397452218, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 27, + "sum_logprob": -19.035940170288086, + "mean_logprob": -0.705034821121781, + "total_sequence_tokens": 35 + }, + { + "index": 2, + "tokens": 13, + "sum_logprob": -21.432252883911133, + "mean_logprob": -1.6486348372239332, + "total_sequence_tokens": 21 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -35.489845275878906, + "mean_logprob": -3.5489845275878906, + "total_sequence_tokens": 18 + } + ], + "best_vs_second_margin": 0.9436000161021523, + "context_truncated": false, + "elapsed_seconds": 0.03724283799965633 + }, + { + "id": "code_completion-027", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def balanced_parentheses(text):\n depth = 0\n for char in text:\n if char == \"(\":\n depth += 1\n elif char == \")\":\n depth -= 1\n if depth < 0:\n return False\n", + "continuations": [ + " return True\n", + " return depth > 0\n", + " return depth == 0\n", + " return len(text) == depth\n" + ], + "tags": [ + "python" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -6.613285541534424, + "mean_logprob": -1.653321385383606, + "total_sequence_tokens": 72 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -13.789587020874023, + "mean_logprob": -1.723698377609253, + "total_sequence_tokens": 76 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -7.251691818237305, + "mean_logprob": -0.9064614772796631, + "total_sequence_tokens": 76 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -9.23131275177002, + "mean_logprob": -0.923131275177002, + "total_sequence_tokens": 78 + } + ], + "best_vs_second_margin": 0.01666979789733891, + "context_truncated": false, + "elapsed_seconds": 0.06129010799850221 + }, + { + "id": "code_completion-028", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def rotate_left(items, amount):\n if not items:\n return []\n", + "continuations": [ + " return items[::-1]\n", + " return items[:amount]\n", + " return items + amount\n", + " amount %= len(items)\n return items[amount:] + items[:amount]\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -15.63742446899414, + "mean_logprob": -1.9546780586242676, + "total_sequence_tokens": 29 + }, + { + "index": 1, + "tokens": 7, + "sum_logprob": -11.773261070251465, + "mean_logprob": -1.6818944386073522, + "total_sequence_tokens": 28 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -18.23381996154785, + "mean_logprob": -2.6048314230782643, + "total_sequence_tokens": 28 + }, + { + "index": 3, + "tokens": 23, + "sum_logprob": -23.40024757385254, + "mean_logprob": -1.0174020684283713, + "total_sequence_tokens": 44 + } + ], + "best_vs_second_margin": 0.6644923701789809, + "context_truncated": false, + "elapsed_seconds": 0.05138291900038894 + }, + { + "id": "code_completion-029", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def group_by_first_letter(words):\n groups = {}\n for word in words:\n", + "continuations": [ + " groups.setdefault(word[0], []).append(word)\n return groups\n", + " groups[word] = word[0]\n return words\n", + " groups.append(word[0])\n return groups\n", + " word[0] = groups\n return groups.keys()\n" + ], + "tags": [ + "python" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 21, + "sum_logprob": -22.006641387939453, + "mean_logprob": -1.047935304187593, + "total_sequence_tokens": 47 + }, + { + "index": 1, + "tokens": 16, + "sum_logprob": -43.108314514160156, + "mean_logprob": -2.6942696571350098, + "total_sequence_tokens": 42 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -30.153905868530273, + "mean_logprob": -2.153850419180734, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 16, + "sum_logprob": -55.40138626098633, + "mean_logprob": -3.4625866413116455, + "total_sequence_tokens": 42 + } + ], + "best_vs_second_margin": 1.105915114993141, + "context_truncated": false, + "elapsed_seconds": 0.03654749700035609 + }, + { + "id": "code_completion-030", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def moving_pairs(items):\n", + "continuations": [ + " return zip(items, items)\n", + " return list(zip(items, items[1:]))\n", + " return items[::2]\n", + " return [(items[0], items[-1])]\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -19.146240234375, + "mean_logprob": -2.1273600260416665, + "total_sequence_tokens": 17 + }, + { + "index": 1, + "tokens": 14, + "sum_logprob": -15.492321968078613, + "mean_logprob": -1.1065944262913294, + "total_sequence_tokens": 22 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -18.950725555419922, + "mean_logprob": -2.3688406944274902, + "total_sequence_tokens": 16 + }, + { + "index": 3, + "tokens": 13, + "sum_logprob": -26.311683654785156, + "mean_logprob": -2.023975665752704, + "total_sequence_tokens": 21 + } + ], + "best_vs_second_margin": 0.9173812394613747, + "context_truncated": false, + "elapsed_seconds": 0.04552672099998745 + }, + { + "id": "code_completion-031", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def invert_mapping(mapping):\n", + "continuations": [ + " return dict(mapping)\n", + " return list(mapping.items())\n", + " return {value: key for key, value in mapping.items()}\n", + " return mapping[::-1]\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 2, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -16.197599411010742, + "mean_logprob": -2.3139427730015347, + "total_sequence_tokens": 15 + }, + { + "index": 1, + "tokens": 9, + "sum_logprob": -14.858826637268066, + "mean_logprob": -1.6509807374742296, + "total_sequence_tokens": 17 + }, + { + "index": 2, + "tokens": 17, + "sum_logprob": -18.217885971069336, + "mean_logprob": -1.0716403512393726, + "total_sequence_tokens": 25 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -8.22993278503418, + "mean_logprob": -1.0287415981292725, + "total_sequence_tokens": 16 + } + ], + "best_vs_second_margin": 0.04289875311010016, + "context_truncated": false, + "elapsed_seconds": 0.04215201799888746 + }, + { + "id": "code_completion-032", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def all_equal(items):\n iterator = iter(items)\n try:\n first = next(iterator)\n except StopIteration:\n", + "continuations": [ + " return False\n return first == iterator\n", + " first = None\n return items == first\n", + " return []\n return len(items)\n", + " return True\n return all(item == first for item in iterator)\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -20.89051055908203, + "mean_logprob": -1.8991373235529119, + "total_sequence_tokens": 47 + }, + { + "index": 1, + "tokens": 13, + "sum_logprob": -26.401443481445312, + "mean_logprob": -2.0308802678034854, + "total_sequence_tokens": 49 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -30.185457229614258, + "mean_logprob": -2.5154547691345215, + "total_sequence_tokens": 48 + }, + { + "index": 3, + "tokens": 18, + "sum_logprob": -34.05173110961914, + "mean_logprob": -1.8917628394232855, + "total_sequence_tokens": 54 + } + ], + "best_vs_second_margin": 0.00737448412962638, + "context_truncated": false, + "elapsed_seconds": 0.042933908000122756 + }, + { + "id": "code_completion-033", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 0.0, + "item_elo": 1050, + "context": "def transpose(rows):\n", + "continuations": [ + " return [list(column) for column in zip(*rows)]\n", + " return rows[::-1]\n", + " return list(zip(rows))\n", + " return [sum(row) for row in rows]\n" + ], + "tags": [ + "python" + ], + "prediction": 3, + "label": 0, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 16, + "sum_logprob": -21.815868377685547, + "mean_logprob": -1.3634917736053467, + "total_sequence_tokens": 22 + }, + { + "index": 1, + "tokens": 8, + "sum_logprob": -10.598787307739258, + "mean_logprob": -1.3248484134674072, + "total_sequence_tokens": 14 + }, + { + "index": 2, + "tokens": 9, + "sum_logprob": -20.984830856323242, + "mean_logprob": -2.3316478729248047, + "total_sequence_tokens": 15 + }, + { + "index": 3, + "tokens": 14, + "sum_logprob": -13.52786636352539, + "mean_logprob": -0.9662761688232422, + "total_sequence_tokens": 20 + } + ], + "best_vs_second_margin": 0.35857224464416504, + "context_truncated": false, + "elapsed_seconds": 0.04276205599853711 + }, + { + "id": "code_completion-034", + "category": "code_completion", + "difficulty": "medium", + "category_weight": 1.4, + "difficulty_weight": 1.5, + "point_weight": 2.0999999999999996, + "earned_points": 2.0999999999999996, + "item_elo": 1050, + "context": "def running_totals(numbers):\n totals = []\n current = 0\n for number in numbers:\n", + "continuations": [ + " totals.append(number)\n return current\n", + " current += number\n totals.append(current)\n return totals\n", + " current = number\n return [current]\n", + " totals += current\n return numbers\n" + ], + "tags": [ + "python" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -39.600975036621094, + "mean_logprob": -3.0462288489708533, + "total_sequence_tokens": 44 + }, + { + "index": 1, + "tokens": 20, + "sum_logprob": -6.991767883300781, + "mean_logprob": -0.34958839416503906, + "total_sequence_tokens": 51 + }, + { + "index": 2, + "tokens": 13, + "sum_logprob": -40.40708923339844, + "mean_logprob": -3.1082376333383412, + "total_sequence_tokens": 44 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -54.728126525878906, + "mean_logprob": -4.975284229625355, + "total_sequence_tokens": 42 + } + ], + "best_vs_second_margin": 2.6966404548058143, + "context_truncated": false, + "elapsed_seconds": 0.057458866000160924 + }, + { + "id": "code_completion-035", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def binary_search(items, target):\n low, high = 0, len(items) - 1\n while low <= high:\n mid = (low + high) // 2\n if items[mid] == target:\n return mid\n if items[mid] < target:\n low = mid + 1\n else:\n", + "continuations": [ + " low = mid - 1\n return low\n", + " high = mid + 1\n return high\n", + " high = mid - 1\n return -1\n", + " return mid\n return None\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 14, + "sum_logprob": -36.48191452026367, + "mean_logprob": -2.605851037161691, + "total_sequence_tokens": 106 + }, + { + "index": 1, + "tokens": 14, + "sum_logprob": -46.85424041748047, + "mean_logprob": -3.3467314583914622, + "total_sequence_tokens": 106 + }, + { + "index": 2, + "tokens": 16, + "sum_logprob": -7.23041296005249, + "mean_logprob": -0.45190081000328064, + "total_sequence_tokens": 108 + }, + { + "index": 3, + "tokens": 8, + "sum_logprob": -23.18140983581543, + "mean_logprob": -2.8976762294769287, + "total_sequence_tokens": 100 + } + ], + "best_vs_second_margin": 2.1539502271584103, + "context_truncated": false, + "elapsed_seconds": 0.07552583399956347 + }, + { + "id": "code_completion-036", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def merge_sorted(left, right):\n result = []\n i = j = 0\n while i < len(left) and j < len(right):\n if left[i] <= right[j]:\n result.append(left[i])\n i += 1\n else:\n result.append(right[j])\n j += 1\n", + "continuations": [ + " return result\n", + " result.extend(left)\n return result\n", + " return sorted(left)\n", + " result.extend(left[i:])\n result.extend(right[j:])\n return result\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 4, + "sum_logprob": -11.00175666809082, + "mean_logprob": -2.750439167022705, + "total_sequence_tokens": 100 + }, + { + "index": 1, + "tokens": 12, + "sum_logprob": -27.22464370727539, + "mean_logprob": -2.2687203089396157, + "total_sequence_tokens": 108 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -48.96931457519531, + "mean_logprob": -6.995616367885044, + "total_sequence_tokens": 103 + }, + { + "index": 3, + "tokens": 24, + "sum_logprob": -6.3913092613220215, + "mean_logprob": -0.26630455255508423, + "total_sequence_tokens": 120 + } + ], + "best_vs_second_margin": 2.0024157563845315, + "context_truncated": false, + "elapsed_seconds": 0.07246739999936835 + }, + { + "id": "code_completion-037", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def topological_visit(node, graph, visiting, visited, order):\n if node in visiting:\n raise ValueError(\"cycle\")\n if node in visited:\n return\n visiting.add(node)\n for neighbor in graph.get(node, []):\n topological_visit(neighbor, graph, visiting, visited, order)\n", + "continuations": [ + " visiting.remove(node)\n visited.add(node)\n order.append(node)\n", + " order.append(node)\n visiting.add(node)\n", + " visited.remove(node)\n order.clear()\n", + " return graph[node]\n" + ], + "tags": [ + "python", + "graph" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 24, + "sum_logprob": -22.875579833984375, + "mean_logprob": -0.953149159749349, + "total_sequence_tokens": 103 + }, + { + "index": 1, + "tokens": 16, + "sum_logprob": -39.46855926513672, + "mean_logprob": -2.466784954071045, + "total_sequence_tokens": 95 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -21.016754150390625, + "mean_logprob": -1.5011967250279017, + "total_sequence_tokens": 93 + }, + { + "index": 3, + "tokens": 7, + "sum_logprob": -17.03171157836914, + "mean_logprob": -2.4331016540527344, + "total_sequence_tokens": 86 + } + ], + "best_vs_second_margin": 0.5480475652785527, + "context_truncated": false, + "elapsed_seconds": 0.07073609600047348 + }, + { + "id": "code_completion-038", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def longest_common_prefix(words):\n if not words:\n return \"\"\n prefix = words[0]\n for word in words[1:]:\n while not word.startswith(prefix):\n", + "continuations": [ + " prefix += word\n", + " prefix = prefix[:-1]\n if not prefix:\n return \"\"\n return prefix\n", + " word = word[:-1]\n return word\n", + " return words[0]\n return prefix + word\n" + ], + "tags": [ + "python", + "string" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -3.5768799781799316, + "mean_logprob": -0.5961466630299886, + "total_sequence_tokens": 58 + }, + { + "index": 1, + "tokens": 25, + "sum_logprob": -27.742603302001953, + "mean_logprob": -1.1097041320800782, + "total_sequence_tokens": 77 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -47.44060516357422, + "mean_logprob": -3.3886146545410156, + "total_sequence_tokens": 66 + }, + { + "index": 3, + "tokens": 14, + "sum_logprob": -72.98765563964844, + "mean_logprob": -5.213403974260602, + "total_sequence_tokens": 66 + } + ], + "best_vs_second_margin": 0.5135574690500896, + "context_truncated": false, + "elapsed_seconds": 0.06210656999974162 + }, + { + "id": "code_completion-039", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def memoized_fibonacci(n, cache=None):\n if cache is None:\n cache = {0: 0, 1: 1}\n if n not in cache:\n", + "continuations": [ + " cache[n] = n\n", + " return cache\n", + " cache[n] = memoized_fibonacci(n - 1, cache) + memoized_fibonacci(n - 2, cache)\n return cache[n]\n", + " cache.clear()\n return n\n" + ], + "tags": [ + "python", + "recursion" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 9, + "sum_logprob": -11.39099407196045, + "mean_logprob": -1.2656660079956055, + "total_sequence_tokens": 56 + }, + { + "index": 1, + "tokens": 4, + "sum_logprob": -30.46306037902832, + "mean_logprob": -7.61576509475708, + "total_sequence_tokens": 51 + }, + { + "index": 2, + "tokens": 45, + "sum_logprob": -11.498723983764648, + "mean_logprob": -0.2555271996392144, + "total_sequence_tokens": 92 + }, + { + "index": 3, + "tokens": 10, + "sum_logprob": -50.775535583496094, + "mean_logprob": -5.077553558349609, + "total_sequence_tokens": 57 + } + ], + "best_vs_second_margin": 1.0101388083563911, + "context_truncated": false, + "elapsed_seconds": 0.0655426350003836 + }, + { + "id": "code_completion-040", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def intervals_overlap(a_start, a_end, b_start, b_end):\n", + "continuations": [ + " return a_end < b_start\n", + " return a_start == b_end\n", + " return a_start + b_start\n", + " return max(a_start, b_start) <= min(a_end, b_end)\n" + ], + "tags": [ + "python", + "interval" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 11, + "sum_logprob": -16.90646743774414, + "mean_logprob": -1.5369515852494673, + "total_sequence_tokens": 33 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -21.989330291748047, + "mean_logprob": -1.9990300265225498, + "total_sequence_tokens": 33 + }, + { + "index": 2, + "tokens": 11, + "sum_logprob": -18.211776733398438, + "mean_logprob": -1.6556160666725852, + "total_sequence_tokens": 33 + }, + { + "index": 3, + "tokens": 25, + "sum_logprob": -22.111881256103516, + "mean_logprob": -0.8844752502441406, + "total_sequence_tokens": 47 + } + ], + "best_vs_second_margin": 0.6524763350053266, + "context_truncated": false, + "elapsed_seconds": 0.04035365600066143 + }, + { + "id": "code_completion-041", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def breadth_first_distances(graph, start):\n from collections import deque\n distances = {start: 0}\n queue = deque([start])\n while queue:\n node = queue.popleft()\n for neighbor in graph.get(node, []):\n if neighbor not in distances:\n", + "continuations": [ + " distances[neighbor] = distances[node] + 1\n queue.append(neighbor)\n return distances\n", + " distances[node] += 1\n return queue\n", + " queue.append(node)\n return graph\n", + " return neighbor\n return distances[start]\n" + ], + "tags": [ + "python", + "graph" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 28, + "sum_logprob": -6.422842025756836, + "mean_logprob": -0.2293872152056013, + "total_sequence_tokens": 107 + }, + { + "index": 1, + "tokens": 14, + "sum_logprob": -47.448028564453125, + "mean_logprob": -3.3891448974609375, + "total_sequence_tokens": 93 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -68.20989990234375, + "mean_logprob": -5.6841583251953125, + "total_sequence_tokens": 91 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -64.69454956054688, + "mean_logprob": -5.881322687322443, + "total_sequence_tokens": 90 + } + ], + "best_vs_second_margin": 3.159757682255336, + "context_truncated": false, + "elapsed_seconds": 0.06932938600039051 + }, + { + "id": "code_completion-042", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def deep_get(mapping, path, default=None):\n current = mapping\n for key in path:\n if not isinstance(current, dict) or key not in current:\n return default\n", + "continuations": [ + " current = mapping[key]\n return mapping\n", + " current = current[key]\n return current\n", + " path = current\n return key\n", + " return current.get(path)\n return default\n" + ], + "tags": [ + "python", + "nested_data" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -32.41277313232422, + "mean_logprob": -2.4932902409480167, + "total_sequence_tokens": 61 + }, + { + "index": 1, + "tokens": 13, + "sum_logprob": -4.214959621429443, + "mean_logprob": -0.32422766318688023, + "total_sequence_tokens": 61 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -39.24943923950195, + "mean_logprob": -3.9249439239501953, + "total_sequence_tokens": 58 + }, + { + "index": 3, + "tokens": 13, + "sum_logprob": -58.77112579345703, + "mean_logprob": -4.520855830265925, + "total_sequence_tokens": 61 + } + ], + "best_vs_second_margin": 2.1690625777611365, + "context_truncated": false, + "elapsed_seconds": 0.04932143499900121 + }, + { + "id": "code_completion-043", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def stable_partition(items, predicate):\n matching = []\n other = []\n for item in items:\n if predicate(item):\n matching.append(item)\n else:\n other.append(item)\n", + "continuations": [ + " return sorted(items)\n", + " return matching or other\n", + " return matching + other\n", + " return zip(matching, other)\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 7, + "sum_logprob": -30.302295684814453, + "mean_logprob": -4.328899383544922, + "total_sequence_tokens": 65 + }, + { + "index": 1, + "tokens": 6, + "sum_logprob": -17.109447479248047, + "mean_logprob": -2.8515745798746743, + "total_sequence_tokens": 64 + }, + { + "index": 2, + "tokens": 7, + "sum_logprob": -12.190431594848633, + "mean_logprob": -1.7414902278355189, + "total_sequence_tokens": 65 + }, + { + "index": 3, + "tokens": 9, + "sum_logprob": -18.88239288330078, + "mean_logprob": -2.0980436537000866, + "total_sequence_tokens": 67 + } + ], + "best_vs_second_margin": 0.35655342586456773, + "context_truncated": false, + "elapsed_seconds": 0.05759788499926799 + }, + { + "id": "code_completion-044", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def parse_key_value(lines):\n result = {}\n for line in lines:\n line = line.strip()\n if not line or line.startswith(\"#\"):\n continue\n key, separator, value = line.partition(\"=\")\n if not separator:\n raise ValueError(\"missing equals sign\")\n", + "continuations": [ + " result[value] = separator\n return lines\n", + " result[key] = key\n return value\n", + " result.clear()\n return key\n", + " result[key.strip()] = value.strip()\n return result\n" + ], + "tags": [ + "python", + "parsing" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -49.231327056884766, + "mean_logprob": -3.787025158221905, + "total_sequence_tokens": 94 + }, + { + "index": 1, + "tokens": 13, + "sum_logprob": -42.809051513671875, + "mean_logprob": -3.293003962590144, + "total_sequence_tokens": 94 + }, + { + "index": 2, + "tokens": 10, + "sum_logprob": -46.267005920410156, + "mean_logprob": -4.626700592041016, + "total_sequence_tokens": 91 + }, + { + "index": 3, + "tokens": 18, + "sum_logprob": -8.798966407775879, + "mean_logprob": -0.48883146709865993, + "total_sequence_tokens": 99 + } + ], + "best_vs_second_margin": 2.804172495491484, + "context_truncated": false, + "elapsed_seconds": 0.07062074500026938 + }, + { + "id": "code_completion-045", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def sliding_window_sums(numbers, width):\n if width <= 0 or width > len(numbers):\n return []\n current = sum(numbers[:width])\n result = [current]\n for index in range(width, len(numbers)):\n", + "continuations": [ + " current += numbers[index] - numbers[index - width]\n result.append(current)\n return result\n", + " current = numbers[index]\n return [current]\n", + " result.append(sum(numbers))\n return numbers\n", + " width += index\n return result[:width]\n" + ], + "tags": [ + "python", + "algorithm" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 30, + "sum_logprob": -41.30909729003906, + "mean_logprob": -1.3769699096679688, + "total_sequence_tokens": 98 + }, + { + "index": 1, + "tokens": 16, + "sum_logprob": -62.10095977783203, + "mean_logprob": -3.881309986114502, + "total_sequence_tokens": 84 + }, + { + "index": 2, + "tokens": 14, + "sum_logprob": -73.57785034179688, + "mean_logprob": -5.255560738699777, + "total_sequence_tokens": 82 + }, + { + "index": 3, + "tokens": 13, + "sum_logprob": -64.48774719238281, + "mean_logprob": -4.960595937875601, + "total_sequence_tokens": 81 + } + ], + "best_vs_second_margin": 2.504340076446533, + "context_truncated": false, + "elapsed_seconds": 0.07155449200035946 + }, + { + "id": "code_completion-046", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 0.0, + "item_elo": 1200, + "context": "def retry(operation, attempts):\n last_error = None\n for _ in range(attempts):\n try:\n return operation()\n except Exception as error:\n last_error = error\n", + "continuations": [ + " return last_error\n", + " if last_error is not None:\n raise last_error\n raise ValueError(\"attempts must be positive\")\n", + " operation()\n return None\n", + " except last_error:\n return attempts\n" + ], + "tags": [ + "python", + "exceptions" + ], + "prediction": 0, + "label": 1, + "passed": false, + "choice_scores": [ + { + "index": 0, + "tokens": 6, + "sum_logprob": -7.404885768890381, + "mean_logprob": -1.2341476281483967, + "total_sequence_tokens": 57 + }, + { + "index": 1, + "tokens": 28, + "sum_logprob": -61.165992736816406, + "mean_logprob": -2.184499740600586, + "total_sequence_tokens": 79 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -27.670310974121094, + "mean_logprob": -3.4587888717651367, + "total_sequence_tokens": 59 + }, + { + "index": 3, + "tokens": 11, + "sum_logprob": -44.15677261352539, + "mean_logprob": -4.014252055775035, + "total_sequence_tokens": 62 + } + ], + "best_vs_second_margin": 0.9503521124521892, + "context_truncated": false, + "elapsed_seconds": 0.05811959799939359 + }, + { + "id": "code_completion-047", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def normalize_rows(matrix):\n result = []\n for row in matrix:\n total = sum(row)\n if total == 0:\n result.append([0.0 for _ in row])\n else:\n", + "continuations": [ + " result.append(row)\n", + " result.append([total for _ in row])\n", + " result.append([value / total for value in row])\n return result\n", + " result = row / total\n return matrix\n" + ], + "tags": [ + "python", + "numeric" + ], + "prediction": 2, + "label": 2, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 8, + "sum_logprob": -12.316730499267578, + "mean_logprob": -1.5395913124084473, + "total_sequence_tokens": 67 + }, + { + "index": 1, + "tokens": 13, + "sum_logprob": -21.836929321289062, + "mean_logprob": -1.6797637939453125, + "total_sequence_tokens": 72 + }, + { + "index": 2, + "tokens": 19, + "sum_logprob": -18.89575958251953, + "mean_logprob": -0.9945136622378701, + "total_sequence_tokens": 78 + }, + { + "index": 3, + "tokens": 13, + "sum_logprob": -36.02602005004883, + "mean_logprob": -2.7712323115422177, + "total_sequence_tokens": 72 + } + ], + "best_vs_second_margin": 0.5450776501705772, + "context_truncated": false, + "elapsed_seconds": 0.04855151400079194 + }, + { + "id": "code_completion-048", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def merge_ranges(ranges):\n if not ranges:\n return []\n ordered = sorted(ranges)\n merged = [list(ordered[0])]\n for start, end in ordered[1:]:\n previous = merged[-1]\n if start <= previous[1]:\n previous[1] = max(previous[1], end)\n else:\n", + "continuations": [ + " previous[0] = start\n return ranges\n", + " merged = [[start, end]]\n return ordered\n", + " return previous\n return merged\n", + " merged.append([start, end])\n return [tuple(pair) for pair in merged]\n" + ], + "tags": [ + "python", + "interval" + ], + "prediction": 3, + "label": 3, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -77.74765014648438, + "mean_logprob": -5.98058847280649, + "total_sequence_tokens": 105 + }, + { + "index": 1, + "tokens": 15, + "sum_logprob": -66.21233367919922, + "mean_logprob": -4.414155578613281, + "total_sequence_tokens": 107 + }, + { + "index": 2, + "tokens": 8, + "sum_logprob": -38.67328643798828, + "mean_logprob": -4.834160804748535, + "total_sequence_tokens": 100 + }, + { + "index": 3, + "tokens": 24, + "sum_logprob": -42.53789520263672, + "mean_logprob": -1.7724123001098633, + "total_sequence_tokens": 116 + } + ], + "best_vs_second_margin": 2.6417432785034176, + "context_truncated": false, + "elapsed_seconds": 0.07347782599936181 + }, + { + "id": "code_completion-049", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def count_paths(grid, row=0, col=0, memo=None):\n if memo is None:\n memo = {}\n if row >= len(grid) or col >= len(grid[0]) or grid[row][col] == 1:\n return 0\n if (row, col) == (len(grid) - 1, len(grid[0]) - 1):\n return 1\n if (row, col) not in memo:\n", + "continuations": [ + " memo[(row, col)] = count_paths(grid, row + 1, col, memo) + count_paths(grid, row, col + 1, memo)\n return memo[(row, col)]\n", + " memo.clear()\n return 0\n", + " return count_paths(grid, row - 1, col - 1, memo)\n return memo\n", + " memo[(row, col)] = 1\n return len(memo)\n" + ], + "tags": [ + "python", + "dynamic_programming" + ], + "prediction": 0, + "label": 0, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 53, + "sum_logprob": -6.174004554748535, + "mean_logprob": -0.11649065197638746, + "total_sequence_tokens": 175 + }, + { + "index": 1, + "tokens": 11, + "sum_logprob": -51.11182403564453, + "mean_logprob": -4.646529457785866, + "total_sequence_tokens": 133 + }, + { + "index": 2, + "tokens": 27, + "sum_logprob": -71.41888427734375, + "mean_logprob": -2.6451438621238426, + "total_sequence_tokens": 149 + }, + { + "index": 3, + "tokens": 20, + "sum_logprob": -37.429500579833984, + "mean_logprob": -1.8714750289916993, + "total_sequence_tokens": 142 + } + ], + "best_vs_second_margin": 1.7549843770153117, + "context_truncated": false, + "elapsed_seconds": 0.10541650500090327 + }, + { + "id": "code_completion-050", + "category": "code_completion", + "difficulty": "hard", + "category_weight": 1.4, + "difficulty_weight": 2.25, + "point_weight": 3.15, + "earned_points": 3.15, + "item_elo": 1200, + "context": "def compose(*functions):\n def composed(value):\n result = value\n for function in reversed(functions):\n", + "continuations": [ + " result = functions(result)\n return value\n", + " result = function(result)\n return result\n return composed\n", + " return function\n return functions\n return result\n", + " result.append(function)\n return result\n return functions\n" + ], + "tags": [ + "python", + "higher_order" + ], + "prediction": 1, + "label": 1, + "passed": true, + "choice_scores": [ + { + "index": 0, + "tokens": 13, + "sum_logprob": -36.70912170410156, + "mean_logprob": -2.823778592623197, + "total_sequence_tokens": 41 + }, + { + "index": 1, + "tokens": 17, + "sum_logprob": -9.565998077392578, + "mean_logprob": -0.562705769258387, + "total_sequence_tokens": 45 + }, + { + "index": 2, + "tokens": 12, + "sum_logprob": -66.07337951660156, + "mean_logprob": -5.506114959716797, + "total_sequence_tokens": 40 + }, + { + "index": 3, + "tokens": 16, + "sum_logprob": -38.47551727294922, + "mean_logprob": -2.404719829559326, + "total_sequence_tokens": 44 + } + ], + "best_vs_second_margin": 1.8420140603009392, + "context_truncated": false, + "elapsed_seconds": 0.0417035800001031 + } + ] +} diff --git a/reports/palmer-006_lm_eval_harness.log b/reports/palmer-006_lm_eval_harness.log new file mode 100644 index 0000000..a2fa074 --- /dev/null +++ b/reports/palmer-006_lm_eval_harness.log @@ -0,0 +1,94 @@ +================================================================================ +RUN 1 — ARC-Easy, ARC-Challenge, Winogrande, PIQA +================================================================================ +2026-07-26:19:07:26 INFO [_cli.run:388] Selected Tasks: ['arc_easy', 'arc_challenge', 'winogrande', 'piqa'] +2026-07-26:19:07:28 INFO [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234 +2026-07-26:19:07:28 INFO [evaluator:239] Initializing hf model, with arguments: {'pretrained': 'appvoid/palmer-006'} +2026-07-26:19:07:49 INFO [models.huggingface:286] Using device 'cuda:0' +config.json: 1.62kB [00:00, 3.38MB/s] +tokenizer_config.json: 99.6kB [00:00, 54.3MB/s] +tokenizer.json: 2.35MB [00:00, 11.4MB/s] +special_tokens_map.json: 7.42kB [00:00, 19.4MB/s] +2026-07-26:19:07:52 INFO [models.huggingface:579] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'} +model.safetensors: 100%|█████████████████████| 182M/182M [00:08<00:00, 22.1MB/s] +[transformers] The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d +Loading weights: 100%|██████████████████████| 386/386 [00:00<00:00, 2314.98it/s] +generation_config.json: 100%|███████████████████| 133/133 [00:00<00:00, 571kB/s] +README.md: 9.00kB [00:00, 161kB/s] +ARC-Easy/train-00000-of-00001.parquet: 100%|██| 331k/331k [00:00<00:00, 803kB/s] +ARC-Easy/test-00000-of-00001.parquet: 100%|██| 346k/346k [00:00<00:00, 1.63MB/s] +ARC-Easy/validation-00000-of-00001.parqu(…): 100%|█| 86.1k/86.1k [00:00<00:00, 4 +Generating train split: 100%|█████| 2251/2251 [00:00<00:00, 24351.27 examples/s] +Generating test split: 100%|█████| 2376/2376 [00:00<00:00, 393371.21 examples/s] +Generating validation split: 100%|█| 570/570 [00:00<00:00, 154851.56 examples/s] +ARC-Challenge/train-00000-of-00001.parqu(…): 100%|█| 190k/190k [00:00<00:00, 896 +ARC-Challenge/test-00000-of-00001.parque(…): 100%|█| 204k/204k [00:00<00:00, 959 +ARC-Challenge/validation-00000-of-00001.(…): 100%|█| 55.7k/55.7k [00:00<00:00, 2 +Generating train split: 100%|████| 1119/1119 [00:00<00:00, 177512.34 examples/s] +Generating test split: 100%|█████| 1172/1172 [00:00<00:00, 224575.10 examples/s] +Generating validation split: 100%|█| 299/299 [00:00<00:00, 101431.32 examples/s] +README.md: 11.2kB [00:00, 25.5MB/s] +winogrande_xl/train-00000-of-00001.parqu(…): 100%|█| 2.06M/2.06M [00:00<00:00, 4 +winogrande_xl/test-00000-of-00001.parque(…): 100%|█| 118k/118k [00:00<00:00, 284 +winogrande_xl/validation-00000-of-00001.(…): 100%|█| 85.9k/85.9k [00:00<00:00, 2 +Generating train split: 100%|█| 40398/40398 [00:00<00:00, 1299756.78 examples/s] +Generating test split: 100%|█████| 1767/1767 [00:00<00:00, 430541.14 examples/s] +Generating validation split: 100%|█| 1267/1267 [00:00<00:00, 403108.79 examples/ +piqa_train.parquet: 100%|██████████████████| 2.64M/2.64M [00:00<00:00, 3.25MB/s] +piqa_validation.parquet: 100%|████████████████| 300k/300k [00:00<00:00, 726kB/s] +piqa_test.parquet: 100%|█████████████████████| 496k/496k [00:00<00:00, 1.20MB/s] +Generating train split: 100%|██| 16113/16113 [00:00<00:00, 686092.14 examples/s] +Generating validation split: 100%|█| 1838/1838 [00:00<00:00, 398137.21 examples/ +Generating test split: 100%|█████| 3084/3084 [00:00<00:00, 528054.93 examples/s] +2026-07-26:19:08:16 INFO [evaluator_utils:446] Selected tasks: +2026-07-26:19:08:16 INFO [evaluator_utils:480] Task: arc_challenge (arc/arc_challenge.yaml) +2026-07-26:19:08:16 INFO [evaluator_utils:480] Task: arc_easy (arc/arc_easy.yaml) +2026-07-26:19:08:16 INFO [evaluator_utils:480] Task: piqa (piqa/piqa.yaml) +2026-07-26:19:08:16 INFO [evaluator_utils:480] Task: winogrande (winogrande/default.yaml) +2026-07-26:19:08:16 INFO [api.task:312] Building contexts for arc_easy on rank 0... +100%|██████████████████████████████████████| 2376/2376 [00:04<00:00, 507.62it/s] +2026-07-26:19:08:21 INFO [api.task:312] Building contexts for arc_challenge on rank 0... +100%|██████████████████████████████████████| 1172/1172 [00:02<00:00, 505.21it/s] +2026-07-26:19:08:24 INFO [api.task:312] Building contexts for winogrande on rank 0... +100%|████████████████████████████████████| 1267/1267 [00:00<00:00, 73584.28it/s] +2026-07-26:19:08:24 INFO [api.task:312] Building contexts for piqa on rank 0... +100%|██████████████████████████████████████| 1838/1838 [00:02<00:00, 691.67it/s] +2026-07-26:19:08:26 INFO [evaluator:585] Running loglikelihood requests +Tokenizing inputs: 0%| | 0/20398 [00:00", + ">>ABSTRACT<<", + ">>INTRODUCTION<<", + ">>SUMMARY<<", + ">>COMMENT<<", + ">>ANSWER<<", + ">>QUESTION<<", + ">>DOMAIN<<", + ">>PREFIX<<", + ">>SUFFIX<<", + ">>MIDDLE<<", + "<|finetune_right_pad_id|>", + "<|start_header_id|>", + "<|end_header_id|>", + "<|eom_id|>", + "<|eot_id|>", + "<|begin_of_text|>", + ">>TITLE<<", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + ">>UNUSED_119<<", + ">>UNUSED_120<<", + ">>UNUSED_215<<", + ">>UNUSED_216<<", + ">>UNUSED_217<<", + ">>UNUSED_218<<", + ">>UNUSED_219<<", + ">>UNUSED_220<<", + ">>UNUSED_221<<", + ">>UNUSED_222<<", + ">>UNUSED_223<<", + ">>UNUSED_224<<", + ">>UNUSED_225<<", + ">>UNUSED_226<<", + ">>UNUSED_227<<", + ">>UNUSED_228<<", + ">>UNUSED_229<<", + ">>UNUSED_230<<", + ">>UNUSED_231<<", + ">>UNUSED_232<<", + ">>UNUSED_233<<", + ">>UNUSED_234<<", + ">>UNUSED_235<<", + ">>UNUSED_236<<", + ">>UNUSED_237<<", + ">>UNUSED_238<<", + ">>UNUSED_239<<", + ">>UNUSED_240<<", + ">>UNUSED_241<<", + ">>UNUSED_242<<", + ">>UNUSED_243<<", + ">>UNUSED_244<<", + ">>UNUSED_245<<", + ">>UNUSED_246<<", + ">>UNUSED_247<<", + ">>UNUSED_248<<", + ">>UNUSED_249<<", + ">>UNUSED_250<<", + ">>UNUSED_251<<", + ">>UNUSED_252<<", + ">>UNUSED_253<<", + ">>UNUSED_254<<", + ">>UNUSED_255<<", + ">>UNUSED_256<<", + ">>UNUSED_257<<", + ">>UNUSED_258<<", + ">>UNUSED_259<<", + ">>UNUSED_260<<", + ">>UNUSED_261<<", + ">>UNUSED_262<<", + ">>UNUSED_263<<", + ">>UNUSED_264<<", + ">>UNUSED_265<<", + ">>UNUSED_266<<", + ">>UNUSED_267<<", + ">>UNUSED_268<<", + ">>UNUSED_269<<", + ">>UNUSED_270<<", + ">>UNUSED_271<<", + ">>UNUSED_272<<", + ">>UNUSED_273<<", + ">>UNUSED_274<<", + ">>UNUSED_275<<", + ">>UNUSED_276<<", + ">>UNUSED_277<<", + ">>UNUSED_278<<", + ">>UNUSED_279<<", + ">>UNUSED_280<<", + ">>UNUSED_281<<", + ">>UNUSED_282<<", + ">>UNUSED_283<<", + ">>UNUSED_284<<", + ">>UNUSED_285<<", + ">>UNUSED_286<<", + ">>UNUSED_287<<", + ">>UNUSED_288<<", + ">>UNUSED_289<<", + ">>UNUSED_290<<", + ">>UNUSED_291<<", + ">>UNUSED_292<<", + ">>UNUSED_293<<", + ">>UNUSED_294<<", + ">>UNUSED_295<<", + ">>UNUSED_296<<", + ">>UNUSED_297<<", + ">>UNUSED_298<<", + ">>UNUSED_299<<", + ">>UNUSED_300<<", + ">>UNUSED_301<<", + ">>UNUSED_302<<", + ">>UNUSED_303<<", + ">>UNUSED_304<<", + ">>UNUSED_305<<", + ">>UNUSED_306<<", + ">>UNUSED_307<<", + ">>UNUSED_308<<", + ">>UNUSED_309<<", + ">>UNUSED_310<<", + ">>UNUSED_311<<", + ">>UNUSED_312<<", + ">>UNUSED_313<<", + ">>UNUSED_314<<", + ">>UNUSED_315<<", + ">>UNUSED_316<<", + ">>UNUSED_317<<", + ">>UNUSED_318<<", + ">>UNUSED_319<<", + ">>UNUSED_320<<", + ">>UNUSED_321<<", + ">>UNUSED_322<<", + ">>UNUSED_323<<", + ">>UNUSED_324<<", + ">>UNUSED_325<<", + ">>UNUSED_326<<", + ">>UNUSED_327<<", + ">>UNUSED_328<<", + ">>UNUSED_329<<", + ">>UNUSED_330<<", + ">>UNUSED_331<<", + ">>UNUSED_332<<", + ">>UNUSED_333<<", + ">>UNUSED_334<<", + ">>UNUSED_335<<", + ">>UNUSED_336<<", + ">>UNUSED_337<<", + ">>UNUSED_338<<", + ">>UNUSED_339<<", + ">>UNUSED_340<<", + ">>UNUSED_341<<", + ">>UNUSED_342<<", + ">>UNUSED_343<<", + ">>UNUSED_344<<", + ">>UNUSED_345<<", + ">>UNUSED_346<<", + ">>UNUSED_347<<", + ">>UNUSED_348<<", + ">>UNUSED_349<<", + ">>UNUSED_350<<", + ">>UNUSED_351<<", + ">>UNUSED_352<<", + ">>UNUSED_353<<", + ">>UNUSED_354<<", + ">>UNUSED_355<<", + ">>UNUSED_356<<", + ">>UNUSED_357<<", + ">>UNUSED_358<<", + ">>UNUSED_359<<", + ">>UNUSED_360<<", + ">>UNUSED_361<<", + ">>UNUSED_362<<", + ">>UNUSED_363<<", + ">>UNUSED_364<<", + ">>UNUSED_365<<", + ">>UNUSED_366<<", + ">>UNUSED_367<<", + ">>UNUSED_368<<", + ">>UNUSED_369<<", + ">>UNUSED_370<<", + ">>UNUSED_371<<", + ">>UNUSED_372<<", + ">>UNUSED_373<<", + ">>UNUSED_374<<", + ">>UNUSED_375<<", + ">>UNUSED_376<<", + ">>UNUSED_377<<", + ">>UNUSED_378<<", + ">>UNUSED_379<<", + ">>UNUSED_380<<", + ">>UNUSED_381<<", + ">>UNUSED_382<<", + ">>UNUSED_383<<", + ">>UNUSED_384<<", + ">>UNUSED_385<<", + ">>UNUSED_386<<", + ">>UNUSED_387<<", + ">>UNUSED_388<<", + ">>UNUSED_389<<", + ">>UNUSED_390<<", + ">>UNUSED_391<<", + ">>UNUSED_392<<", + ">>UNUSED_393<<", + ">>UNUSED_394<<", + ">>UNUSED_395<<", + ">>UNUSED_396<<", + ">>UNUSED_397<<", + ">>UNUSED_398<<", + ">>UNUSED_399<<", + ">>UNUSED_400<<", + ">>UNUSED_401<<", + ">>UNUSED_402<<", + ">>UNUSED_403<<", + ">>UNUSED_404<<", + ">>UNUSED_405<<", + ">>UNUSED_406<<", + ">>UNUSED_407<<", + ">>UNUSED_408<<", + ">>UNUSED_409<<", + ">>UNUSED_410<<", + ">>UNUSED_411<<", + ">>UNUSED_412<<", + ">>UNUSED_413<<", + ">>UNUSED_414<<", + ">>UNUSED_415<<", + ">>UNUSED_416<<", + ">>UNUSED_417<<", + ">>UNUSED_418<<", + ">>UNUSED_419<<", + ">>UNUSED_420<<", + ">>UNUSED_421<<", + ">>UNUSED_422<<", + ">>UNUSED_423<<", + ">>UNUSED_424<<", + ">>UNUSED_425<<", + ">>UNUSED_426<<", + ">>UNUSED_427<<", + ">>UNUSED_428<<", + ">>UNUSED_429<<", + ">>UNUSED_430<<", + ">>UNUSED_431<<", + ">>UNUSED_432<<", + ">>UNUSED_433<<", + ">>UNUSED_434<<", + ">>UNUSED_435<<", + ">>UNUSED_436<<", + ">>UNUSED_437<<", + ">>UNUSED_438<<", + ">>UNUSED_439<<", + ">>UNUSED_440<<", + ">>UNUSED_441<<", + ">>UNUSED_442<<", + ">>UNUSED_443<<", + ">>UNUSED_444<<", + ">>UNUSED_445<<", + ">>UNUSED_446<<", + ">>UNUSED_447<<", + ">>UNUSED_448<<", + ">>UNUSED_449<<", + ">>UNUSED_450<<", + ">>UNUSED_451<<", + ">>UNUSED_452<<", + ">>UNUSED_453<<", + ">>UNUSED_454<<", + ">>UNUSED_455<<", + ">>UNUSED_456<<", + ">>UNUSED_457<<", + ">>UNUSED_458<<", + ">>UNUSED_459<<", + ">>UNUSED_460<<", + ">>UNUSED_461<<", + ">>UNUSED_462<<", + ">>UNUSED_463<<", + ">>UNUSED_464<<", + ">>UNUSED_465<<", + ">>UNUSED_466<<", + ">>UNUSED_467<<", + ">>UNUSED_468<<", + ">>UNUSED_469<<", + ">>UNUSED_470<<", + ">>UNUSED_471<<", + ">>UNUSED_472<<", + ">>UNUSED_473<<", + ">>UNUSED_474<<", + ">>UNUSED_475<<", + ">>UNUSED_476<<", + ">>UNUSED_477<<", + ">>UNUSED_478<<", + ">>UNUSED_479<<", + ">>UNUSED_480<<", + ">>UNUSED_481<<", + ">>UNUSED_482<<", + ">>UNUSED_483<<", + ">>UNUSED_484<<", + ">>UNUSED_485<<", + ">>UNUSED_486<<", + ">>UNUSED_487<<", + ">>UNUSED_488<<", + ">>UNUSED_489<<", + ">>UNUSED_490<<", + ">>UNUSED_491<<", + ">>UNUSED_492<<", + ">>UNUSED_493<<", + ">>UNUSED_494<<", + ">>UNUSED_495<<", + ">>UNUSED_496<<", + ">>UNUSED_497<<", + ">>UNUSED_498<<", + ">>UNUSED_499<<", + ">>UNUSED_500<<", + ">>UNUSED_501<<", + ">>UNUSED_502<<", + ">>UNUSED_503<<", + ">>UNUSED_504<<", + ">>UNUSED_505<<", + ">>UNUSED_506<<", + ">>UNUSED_507<<", + ">>UNUSED_508<<", + ">>UNUSED_509<<", + ">>UNUSED_510<<", + ">>UNUSED_511<<" + ], + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..319e899 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,165548 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "<|pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": ">>ABSTRACT<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": ">>INTRODUCTION<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": ">>SUMMARY<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4, + "content": ">>COMMENT<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 5, + "content": ">>ANSWER<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 6, + "content": ">>QUESTION<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 7, + "content": ">>DOMAIN<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 8, + "content": ">>PREFIX<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 9, + "content": ">>SUFFIX<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 10, + "content": ">>MIDDLE<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 11, + "content": "<|end_of_text|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 12, + "content": "<|finetune_right_pad_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 13, + "content": "<|start_header_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 14, + "content": "<|end_header_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 15, + "content": "<|eom_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 16, + "content": "<|eot_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 17, + "content": "<|begin_of_text|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 18, + "content": ">>TITLE<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 19, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 20, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 21, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 22, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 23, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 24, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 25, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 26, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 27, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 28, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 29, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 30, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 31, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 32, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 33, + "content": "\\blindtext", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 34, + "content": "\\newpage", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 35, + "content": "\\boxed", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 36, + "content": "\\framebox", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 37, + "content": "\\fbox", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 38, + "content": "\\tag", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 39, + "content": "\\nonumber", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 40, + "content": "\\item", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 41, + "content": "\\centering", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 42, + "content": "\\caption", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 43, + "content": "\\includegraphics", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 44, + "content": "\\label", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 45, + "content": "\\multicolumn", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 46, + "content": "\\cline", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 47, + "content": "\\hline", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 48, + "content": "\\end", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 49, + "content": "\\begin", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 50, + "content": "\\tableofcontents", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 51, + "content": "\\maketitle", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 52, + "content": "\\date", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 53, + "content": "\\author", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 54, + "content": "\\title", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 55, + "content": "\\chapter", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 56, + "content": "\\subsubsection", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 57, + "content": "\\subsection", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 58, + "content": "\\section", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 59, + "content": "\\noindent", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 60, + "content": "\\newline", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 61, + "content": "\\par", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 62, + "content": "\\ddot", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 63, + "content": "\\dot", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 64, + "content": "\\bar", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 65, + "content": "\\iff", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 66, + "content": "\\implies", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 67, + "content": "\\neg", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 68, + "content": "\\lor", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 69, + "content": "\\land", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 70, + "content": "\\qquad", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 71, + "content": "\\quad", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 72, + "content": "\\arctan", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 73, + "content": "\\arccos", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 74, + "content": "\\arcsin", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 75, + "content": "\\cot", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 76, + "content": "\\sec", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 77, + "content": "\\csc", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 78, + "content": "\\tan", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 79, + "content": "\\cos", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 80, + "content": "\\sin", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 81, + "content": "\\widetilde", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 82, + "content": "\\widehat", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 83, + "content": "\\complement", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 84, + "content": "\\varnothing", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 85, + "content": "\\tilde", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 86, + "content": "\\hat", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 87, + "content": "\\vec", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 88, + "content": "\\mathfrak", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 89, + "content": "\\mathcal", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 90, + "content": "\\mathbb", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 91, + "content": "\\emptyset", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 92, + "content": "\\bot", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 93, + "content": "\\aleph", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 94, + "content": "\\Re", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 95, + "content": "\\Im", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 96, + "content": "\\ell", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 97, + "content": "\\hbar", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 98, + "content": "\\exists", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 99, + "content": "\\forall", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 100, + "content": "\\partial", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 101, + "content": "\\nabla", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 102, + "content": "\\infty", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 103, + "content": "\\binom", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 104, + "content": "\\overline", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 105, + "content": "\\sqrt", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 106, + "content": "\\tfrac", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 107, + "content": "\\dfrac", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 108, + "content": "\\frac", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 109, + "content": "\\textsc", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 110, + "content": "\\textsf", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 111, + "content": "\\texttt", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 112, + "content": "\\emph", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 113, + "content": "\\underline", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 114, + "content": "\\textit", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 115, + "content": "\\textbf", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 116, + "content": "\\bigoplus", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 117, + "content": "\\bigcap", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 118, + "content": "\\bigcup", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 119, + "content": "\\prod", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 120, + "content": "\\sum", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 121, + "content": "\\oint", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 122, + "content": "\\iiint", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 123, + "content": "\\iint", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 124, + "content": "\\int", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 125, + "content": "\\right", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 126, + "content": "\\left", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 127, + "content": "\\Bigr", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 128, + "content": "\\Bigl", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 129, + "content": "\\bigr", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 130, + "content": "\\bigl", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 131, + "content": ">>UNUSED_119<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 132, + "content": ">>UNUSED_120<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 133, + "content": "\\rceil", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 134, + "content": "\\lceil", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 135, + "content": "\\rfloor", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 136, + "content": "\\lfloor", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 137, + "content": "\\rangle", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 138, + "content": "\\langle", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 139, + "content": "\\overleftarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 140, + "content": "\\overleftrightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 141, + "content": "\\overrightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 142, + "content": "\\Longleftrightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 143, + "content": "\\longleftrightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 144, + "content": "\\longrightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 145, + "content": "\\longleftarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 146, + "content": "\\Longrightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 147, + "content": "\\Longleftarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 148, + "content": "\\gets", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 149, + "content": "\\to", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 150, + "content": "\\mapsto", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 151, + "content": "\\Updownarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 152, + "content": "\\Downarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 153, + "content": "\\Uparrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 154, + "content": "\\updownarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 155, + "content": "\\downarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 156, + "content": "\\uparrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 157, + "content": "\\Leftrightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 158, + "content": "\\Rightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 159, + "content": "\\Leftarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 160, + "content": "\\leftrightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 161, + "content": "\\rightarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 162, + "content": "\\leftarrow", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 163, + "content": "\\perp", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 164, + "content": "\\propto", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 165, + "content": "\\ni", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 166, + "content": "\\notin", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 167, + "content": "\\in", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 168, + "content": "\\supseteq", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 169, + "content": "\\supset", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 170, + "content": "\\sqsupseteq", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 171, + "content": "\\sqsubseteq", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 172, + "content": "\\subseteq", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 173, + "content": "\\subset", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 174, + "content": "\\cong", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 175, + "content": "\\approx", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 176, + "content": "\\simeq", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 177, + "content": "\\sim", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 178, + "content": "\\equiv", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 179, + "content": "\\neq", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 180, + "content": "\\geq", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 181, + "content": "\\leq", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 182, + "content": "\\oslash", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 183, + "content": "\\otimes", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 184, + "content": "\\ominus", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 185, + "content": "\\oplus", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 186, + "content": "\\wedge", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 187, + "content": "\\vee", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 188, + "content": "\\sqcup", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 189, + "content": "\\sqcap", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 190, + "content": "\\uplus", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 191, + "content": "\\cup", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 192, + "content": "\\cap", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 193, + "content": "\\cdot", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 194, + "content": "\\bullet", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 195, + "content": "\\circ", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 196, + "content": "\\star", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 197, + "content": "\\ast", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 198, + "content": "\\div", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 199, + "content": "\\times", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 200, + "content": "\\mp", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 201, + "content": "\\pm", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 202, + "content": "\\omega", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 203, + "content": "\\psi", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 204, + "content": "\\chi", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 205, + "content": "\\varphi", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 206, + "content": "\\varsigma", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 207, + "content": "\\phi", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 208, + "content": "\\upsilon", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 209, + "content": "\\tau", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 210, + "content": "\\sigma", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 211, + "content": "\\rho", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 212, + "content": "\\pi", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 213, + "content": "\\xi", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 214, + "content": "\\nu", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 215, + "content": "\\mu", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 216, + "content": "\\lambda", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 217, + "content": "\\kappa", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 218, + "content": "\\iota", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 219, + "content": "\\theta", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 220, + "content": "\\eta", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 221, + "content": "\\zeta", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 222, + "content": "\\epsilon", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 223, + "content": "\\delta", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 224, + "content": "\\gamma", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 225, + "content": "\\beta", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 226, + "content": "\\alpha", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 227, + "content": ">>UNUSED_215<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 228, + "content": ">>UNUSED_216<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 229, + "content": ">>UNUSED_217<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 230, + "content": ">>UNUSED_218<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 231, + "content": ">>UNUSED_219<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 232, + "content": ">>UNUSED_220<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 233, + "content": ">>UNUSED_221<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 234, + "content": ">>UNUSED_222<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 235, + "content": ">>UNUSED_223<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 236, + "content": ">>UNUSED_224<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 237, + "content": ">>UNUSED_225<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 238, + "content": ">>UNUSED_226<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 239, + "content": ">>UNUSED_227<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 240, + "content": ">>UNUSED_228<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 241, + "content": ">>UNUSED_229<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 242, + "content": ">>UNUSED_230<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 243, + "content": ">>UNUSED_231<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 244, + "content": ">>UNUSED_232<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 245, + "content": ">>UNUSED_233<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 246, + "content": ">>UNUSED_234<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 247, + "content": ">>UNUSED_235<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 248, + "content": ">>UNUSED_236<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 249, + "content": ">>UNUSED_237<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 250, + "content": ">>UNUSED_238<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 251, + "content": ">>UNUSED_239<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 252, + "content": ">>UNUSED_240<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 253, + "content": ">>UNUSED_241<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 254, + "content": ">>UNUSED_242<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 255, + "content": ">>UNUSED_243<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 256, + "content": ">>UNUSED_244<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 257, + "content": ">>UNUSED_245<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 258, + "content": ">>UNUSED_246<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 259, + "content": ">>UNUSED_247<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 260, + "content": ">>UNUSED_248<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 261, + "content": ">>UNUSED_249<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 262, + "content": ">>UNUSED_250<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 263, + "content": ">>UNUSED_251<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 264, + "content": ">>UNUSED_252<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 265, + "content": ">>UNUSED_253<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 266, + "content": ">>UNUSED_254<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 267, + "content": ">>UNUSED_255<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 268, + "content": ">>UNUSED_256<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 269, + "content": ">>UNUSED_257<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 270, + "content": ">>UNUSED_258<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 271, + "content": ">>UNUSED_259<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 272, + "content": ">>UNUSED_260<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 273, + "content": ">>UNUSED_261<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 274, + "content": ">>UNUSED_262<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 275, + "content": ">>UNUSED_263<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 276, + "content": ">>UNUSED_264<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 277, + "content": ">>UNUSED_265<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 278, + "content": ">>UNUSED_266<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 279, + "content": ">>UNUSED_267<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 280, + "content": ">>UNUSED_268<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 281, + "content": ">>UNUSED_269<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 282, + "content": ">>UNUSED_270<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 283, + "content": ">>UNUSED_271<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 284, + "content": ">>UNUSED_272<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 285, + "content": ">>UNUSED_273<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 286, + "content": ">>UNUSED_274<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 287, + "content": ">>UNUSED_275<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 288, + "content": ">>UNUSED_276<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 289, + "content": ">>UNUSED_277<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 290, + "content": ">>UNUSED_278<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 291, + "content": ">>UNUSED_279<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 292, + "content": ">>UNUSED_280<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 293, + "content": ">>UNUSED_281<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 294, + "content": ">>UNUSED_282<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 295, + "content": ">>UNUSED_283<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 296, + "content": ">>UNUSED_284<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 297, + "content": ">>UNUSED_285<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 298, + "content": ">>UNUSED_286<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 299, + "content": ">>UNUSED_287<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 300, + "content": ">>UNUSED_288<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 301, + "content": ">>UNUSED_289<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 302, + "content": ">>UNUSED_290<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 303, + "content": ">>UNUSED_291<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 304, + "content": ">>UNUSED_292<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 305, + "content": ">>UNUSED_293<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 306, + "content": ">>UNUSED_294<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 307, + "content": ">>UNUSED_295<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 308, + "content": ">>UNUSED_296<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 309, + "content": ">>UNUSED_297<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 310, + "content": ">>UNUSED_298<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 311, + "content": ">>UNUSED_299<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 312, + "content": ">>UNUSED_300<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 313, + "content": ">>UNUSED_301<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 314, + "content": ">>UNUSED_302<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 315, + "content": ">>UNUSED_303<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 316, + "content": ">>UNUSED_304<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 317, + "content": ">>UNUSED_305<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 318, + "content": ">>UNUSED_306<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 319, + "content": ">>UNUSED_307<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 320, + "content": ">>UNUSED_308<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 321, + "content": ">>UNUSED_309<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 322, + "content": ">>UNUSED_310<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 323, + "content": ">>UNUSED_311<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 324, + "content": ">>UNUSED_312<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 325, + "content": ">>UNUSED_313<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 326, + "content": ">>UNUSED_314<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 327, + "content": ">>UNUSED_315<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 328, + "content": ">>UNUSED_316<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 329, + "content": ">>UNUSED_317<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 330, + "content": ">>UNUSED_318<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 331, + "content": ">>UNUSED_319<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 332, + "content": ">>UNUSED_320<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 333, + "content": ">>UNUSED_321<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 334, + "content": ">>UNUSED_322<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 335, + "content": ">>UNUSED_323<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 336, + "content": ">>UNUSED_324<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 337, + "content": ">>UNUSED_325<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 338, + "content": ">>UNUSED_326<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 339, + "content": ">>UNUSED_327<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 340, + "content": ">>UNUSED_328<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 341, + "content": ">>UNUSED_329<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 342, + "content": ">>UNUSED_330<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 343, + "content": ">>UNUSED_331<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 344, + "content": ">>UNUSED_332<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 345, + "content": ">>UNUSED_333<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 346, + "content": ">>UNUSED_334<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 347, + "content": ">>UNUSED_335<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 348, + "content": ">>UNUSED_336<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 349, + "content": ">>UNUSED_337<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 350, + "content": ">>UNUSED_338<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 351, + "content": ">>UNUSED_339<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 352, + "content": ">>UNUSED_340<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 353, + "content": ">>UNUSED_341<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 354, + "content": ">>UNUSED_342<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 355, + "content": ">>UNUSED_343<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 356, + "content": ">>UNUSED_344<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 357, + "content": ">>UNUSED_345<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 358, + "content": ">>UNUSED_346<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 359, + "content": ">>UNUSED_347<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 360, + "content": ">>UNUSED_348<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 361, + "content": ">>UNUSED_349<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 362, + "content": ">>UNUSED_350<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 363, + "content": ">>UNUSED_351<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 364, + "content": ">>UNUSED_352<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 365, + "content": ">>UNUSED_353<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 366, + "content": ">>UNUSED_354<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 367, + "content": ">>UNUSED_355<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 368, + "content": ">>UNUSED_356<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 369, + "content": ">>UNUSED_357<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 370, + "content": ">>UNUSED_358<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 371, + "content": ">>UNUSED_359<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 372, + "content": ">>UNUSED_360<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 373, + "content": ">>UNUSED_361<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 374, + "content": ">>UNUSED_362<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 375, + "content": ">>UNUSED_363<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 376, + "content": ">>UNUSED_364<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 377, + "content": ">>UNUSED_365<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 378, + "content": ">>UNUSED_366<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 379, + "content": ">>UNUSED_367<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 380, + "content": ">>UNUSED_368<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 381, + "content": ">>UNUSED_369<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 382, + "content": ">>UNUSED_370<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 383, + "content": ">>UNUSED_371<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 384, + "content": ">>UNUSED_372<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 385, + "content": ">>UNUSED_373<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 386, + "content": ">>UNUSED_374<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 387, + "content": ">>UNUSED_375<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 388, + "content": ">>UNUSED_376<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 389, + "content": ">>UNUSED_377<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 390, + "content": ">>UNUSED_378<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 391, + "content": ">>UNUSED_379<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 392, + "content": ">>UNUSED_380<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 393, + "content": ">>UNUSED_381<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 394, + "content": ">>UNUSED_382<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 395, + "content": ">>UNUSED_383<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 396, + "content": ">>UNUSED_384<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 397, + "content": ">>UNUSED_385<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 398, + "content": ">>UNUSED_386<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 399, + "content": ">>UNUSED_387<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 400, + "content": ">>UNUSED_388<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 401, + "content": ">>UNUSED_389<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 402, + "content": ">>UNUSED_390<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 403, + "content": ">>UNUSED_391<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 404, + "content": ">>UNUSED_392<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 405, + "content": ">>UNUSED_393<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 406, + "content": ">>UNUSED_394<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 407, + "content": ">>UNUSED_395<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 408, + "content": ">>UNUSED_396<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 409, + "content": ">>UNUSED_397<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 410, + "content": ">>UNUSED_398<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 411, + "content": ">>UNUSED_399<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 412, + "content": ">>UNUSED_400<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 413, + "content": ">>UNUSED_401<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 414, + "content": ">>UNUSED_402<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 415, + "content": ">>UNUSED_403<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 416, + "content": ">>UNUSED_404<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 417, + "content": ">>UNUSED_405<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 418, + "content": ">>UNUSED_406<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 419, + "content": ">>UNUSED_407<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 420, + "content": ">>UNUSED_408<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 421, + "content": ">>UNUSED_409<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 422, + "content": ">>UNUSED_410<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 423, + "content": ">>UNUSED_411<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 424, + "content": ">>UNUSED_412<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 425, + "content": ">>UNUSED_413<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 426, + "content": ">>UNUSED_414<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 427, + "content": ">>UNUSED_415<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 428, + "content": ">>UNUSED_416<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 429, + "content": ">>UNUSED_417<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 430, + "content": ">>UNUSED_418<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 431, + "content": ">>UNUSED_419<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 432, + "content": ">>UNUSED_420<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 433, + "content": ">>UNUSED_421<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 434, + "content": ">>UNUSED_422<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 435, + "content": ">>UNUSED_423<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 436, + "content": ">>UNUSED_424<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 437, + "content": ">>UNUSED_425<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 438, + "content": ">>UNUSED_426<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 439, + "content": ">>UNUSED_427<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 440, + "content": ">>UNUSED_428<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 441, + "content": ">>UNUSED_429<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 442, + "content": ">>UNUSED_430<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 443, + "content": ">>UNUSED_431<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 444, + "content": ">>UNUSED_432<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 445, + "content": ">>UNUSED_433<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 446, + "content": ">>UNUSED_434<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 447, + "content": ">>UNUSED_435<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 448, + "content": ">>UNUSED_436<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 449, + "content": ">>UNUSED_437<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 450, + "content": ">>UNUSED_438<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 451, + "content": ">>UNUSED_439<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 452, + "content": ">>UNUSED_440<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 453, + "content": ">>UNUSED_441<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 454, + "content": ">>UNUSED_442<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 455, + "content": ">>UNUSED_443<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 456, + "content": ">>UNUSED_444<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 457, + "content": ">>UNUSED_445<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 458, + "content": ">>UNUSED_446<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 459, + "content": ">>UNUSED_447<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 460, + "content": ">>UNUSED_448<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 461, + "content": ">>UNUSED_449<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 462, + "content": ">>UNUSED_450<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 463, + "content": ">>UNUSED_451<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 464, + "content": ">>UNUSED_452<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 465, + "content": ">>UNUSED_453<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 466, + "content": ">>UNUSED_454<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 467, + "content": ">>UNUSED_455<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 468, + "content": ">>UNUSED_456<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 469, + "content": ">>UNUSED_457<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 470, + "content": ">>UNUSED_458<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 471, + "content": ">>UNUSED_459<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 472, + "content": ">>UNUSED_460<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 473, + "content": ">>UNUSED_461<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 474, + "content": ">>UNUSED_462<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 475, + "content": ">>UNUSED_463<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 476, + "content": ">>UNUSED_464<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 477, + "content": ">>UNUSED_465<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 478, + "content": ">>UNUSED_466<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 479, + "content": ">>UNUSED_467<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 480, + "content": ">>UNUSED_468<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 481, + "content": ">>UNUSED_469<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 482, + "content": ">>UNUSED_470<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 483, + "content": ">>UNUSED_471<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 484, + "content": ">>UNUSED_472<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 485, + "content": ">>UNUSED_473<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 486, + "content": ">>UNUSED_474<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 487, + "content": ">>UNUSED_475<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 488, + "content": ">>UNUSED_476<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 489, + "content": ">>UNUSED_477<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 490, + "content": ">>UNUSED_478<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 491, + "content": ">>UNUSED_479<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 492, + "content": ">>UNUSED_480<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 493, + "content": ">>UNUSED_481<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 494, + "content": ">>UNUSED_482<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 495, + "content": ">>UNUSED_483<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 496, + "content": ">>UNUSED_484<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 497, + "content": ">>UNUSED_485<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 498, + "content": ">>UNUSED_486<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 499, + "content": ">>UNUSED_487<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 500, + "content": ">>UNUSED_488<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 501, + "content": ">>UNUSED_489<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 502, + "content": ">>UNUSED_490<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 503, + "content": ">>UNUSED_491<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 504, + "content": ">>UNUSED_492<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 505, + "content": ">>UNUSED_493<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 506, + "content": ">>UNUSED_494<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 507, + "content": ">>UNUSED_495<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 508, + "content": ">>UNUSED_496<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 509, + "content": ">>UNUSED_497<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 510, + "content": ">>UNUSED_498<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 511, + "content": ">>UNUSED_499<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 512, + "content": ">>UNUSED_500<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 513, + "content": ">>UNUSED_501<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 514, + "content": ">>UNUSED_502<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 515, + "content": ">>UNUSED_503<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 516, + "content": ">>UNUSED_504<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 517, + "content": ">>UNUSED_505<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 518, + "content": ">>UNUSED_506<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 519, + "content": ">>UNUSED_507<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 520, + "content": ">>UNUSED_508<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 521, + "content": ">>UNUSED_509<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 522, + "content": ">>UNUSED_510<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 523, + "content": ">>UNUSED_511<<", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "Sequence", + "pretokenizers": [ + { + "type": "Punctuation", + "behavior": "Contiguous" + }, + { + "type": "Split", + "pattern": { + "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" + }, + "behavior": "Isolated", + "invert": false + }, + { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": false + }, + { + "type": "Split", + "pattern": { + "Regex": "[0-9]" + }, + "behavior": "Isolated", + "invert": false + } + ] + }, + "post_processor": null, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": { + "<|pad|>": 0, + ">>ABSTRACT<<": 1, + ">>INTRODUCTION<<": 2, + ">>SUMMARY<<": 3, + ">>COMMENT<<": 4, + ">>ANSWER<<": 5, + ">>QUESTION<<": 6, + ">>DOMAIN<<": 7, + ">>PREFIX<<": 8, + ">>SUFFIX<<": 9, + ">>MIDDLE<<": 10, + "<|end_of_text|>": 11, + "<|finetune_right_pad_id|>": 12, + "<|start_header_id|>": 13, + "<|end_header_id|>": 14, + "<|eom_id|>": 15, + "<|eot_id|>": 16, + "<|begin_of_text|>": 17, + ">>TITLE<<": 18, + "": 19, + "": 20, + "": 21, + "": 22, + "": 23, + "": 24, + "": 25, + "": 26, + "": 27, + "": 28, + "": 29, + "": 30, + "": 31, + "": 32, + "\\blindtext": 33, + "\\newpage": 34, + "\\boxed": 35, + "\\framebox": 36, + "\\fbox": 37, + "\\tag": 38, + "\\nonumber": 39, + "\\item": 40, + "\\centering": 41, + "\\caption": 42, + "\\includegraphics": 43, + "\\label": 44, + "\\multicolumn": 45, + "\\cline": 46, + "\\hline": 47, + "\\end": 48, + "\\begin": 49, + "\\tableofcontents": 50, + "\\maketitle": 51, + "\\date": 52, + "\\author": 53, + "\\title": 54, + "\\chapter": 55, + "\\subsubsection": 56, + "\\subsection": 57, + "\\section": 58, + "\\noindent": 59, + "\\newline": 60, + "\\par": 61, + "\\ddot": 62, + "\\dot": 63, + "\\bar": 64, + "\\iff": 65, + "\\implies": 66, + "\\neg": 67, + "\\lor": 68, + "\\land": 69, + "\\qquad": 70, + "\\quad": 71, + "\\arctan": 72, + "\\arccos": 73, + "\\arcsin": 74, + "\\cot": 75, + "\\sec": 76, + "\\csc": 77, + "\\tan": 78, + "\\cos": 79, + "\\sin": 80, + "\\widetilde": 81, + "\\widehat": 82, + "\\complement": 83, + "\\varnothing": 84, + "\\tilde": 85, + "\\hat": 86, + "\\vec": 87, + "\\mathfrak": 88, + "\\mathcal": 89, + "\\mathbb": 90, + "\\emptyset": 91, + "\\bot": 92, + "\\aleph": 93, + "\\Re": 94, + "\\Im": 95, + "\\ell": 96, + "\\hbar": 97, + "\\exists": 98, + "\\forall": 99, + "\\partial": 100, + "\\nabla": 101, + "\\infty": 102, + "\\binom": 103, + "\\overline": 104, + "\\sqrt": 105, + "\\tfrac": 106, + "\\dfrac": 107, + "\\frac": 108, + "\\textsc": 109, + "\\textsf": 110, + "\\texttt": 111, + "\\emph": 112, + "\\underline": 113, + "\\textit": 114, + "\\textbf": 115, + "\\bigoplus": 116, + "\\bigcap": 117, + "\\bigcup": 118, + "\\prod": 119, + "\\sum": 120, + "\\oint": 121, + "\\iiint": 122, + "\\iint": 123, + "\\int": 124, + "\\right": 125, + "\\left": 126, + "\\Bigr": 127, + "\\Bigl": 128, + "\\bigr": 129, + "\\bigl": 130, + ">>UNUSED_119<<": 131, + ">>UNUSED_120<<": 132, + "\\rceil": 133, + "\\lceil": 134, + "\\rfloor": 135, + "\\lfloor": 136, + "\\rangle": 137, + "\\langle": 138, + "\\overleftarrow": 139, + "\\overleftrightarrow": 140, + "\\overrightarrow": 141, + "\\Longleftrightarrow": 142, + "\\longleftrightarrow": 143, + "\\longrightarrow": 144, + "\\longleftarrow": 145, + "\\Longrightarrow": 146, + "\\Longleftarrow": 147, + "\\gets": 148, + "\\to": 149, + "\\mapsto": 150, + "\\Updownarrow": 151, + "\\Downarrow": 152, + "\\Uparrow": 153, + "\\updownarrow": 154, + "\\downarrow": 155, + "\\uparrow": 156, + "\\Leftrightarrow": 157, + "\\Rightarrow": 158, + "\\Leftarrow": 159, + "\\leftrightarrow": 160, + "\\rightarrow": 161, + "\\leftarrow": 162, + "\\perp": 163, + "\\propto": 164, + "\\ni": 165, + "\\notin": 166, + "\\in": 167, + "\\supseteq": 168, + "\\supset": 169, + "\\sqsupseteq": 170, + "\\sqsubseteq": 171, + "\\subseteq": 172, + "\\subset": 173, + "\\cong": 174, + "\\approx": 175, + "\\simeq": 176, + "\\sim": 177, + "\\equiv": 178, + "\\neq": 179, + "\\geq": 180, + "\\leq": 181, + "\\oslash": 182, + "\\otimes": 183, + "\\ominus": 184, + "\\oplus": 185, + "\\wedge": 186, + "\\vee": 187, + "\\sqcup": 188, + "\\sqcap": 189, + "\\uplus": 190, + "\\cup": 191, + "\\cap": 192, + "\\cdot": 193, + "\\bullet": 194, + "\\circ": 195, + "\\star": 196, + "\\ast": 197, + "\\div": 198, + "\\times": 199, + "\\mp": 200, + "\\pm": 201, + "\\omega": 202, + "\\psi": 203, + "\\chi": 204, + "\\varphi": 205, + "\\varsigma": 206, + "\\phi": 207, + "\\upsilon": 208, + "\\tau": 209, + "\\sigma": 210, + "\\rho": 211, + "\\pi": 212, + "\\xi": 213, + "\\nu": 214, + "\\mu": 215, + "\\lambda": 216, + "\\kappa": 217, + "\\iota": 218, + "\\theta": 219, + "\\eta": 220, + "\\zeta": 221, + "\\epsilon": 222, + "\\delta": 223, + "\\gamma": 224, + "\\beta": 225, + "\\alpha": 226, + ">>UNUSED_215<<": 227, + ">>UNUSED_216<<": 228, + ">>UNUSED_217<<": 229, + ">>UNUSED_218<<": 230, + ">>UNUSED_219<<": 231, + ">>UNUSED_220<<": 232, + ">>UNUSED_221<<": 233, + ">>UNUSED_222<<": 234, + ">>UNUSED_223<<": 235, + ">>UNUSED_224<<": 236, + ">>UNUSED_225<<": 237, + ">>UNUSED_226<<": 238, + ">>UNUSED_227<<": 239, + ">>UNUSED_228<<": 240, + ">>UNUSED_229<<": 241, + ">>UNUSED_230<<": 242, + ">>UNUSED_231<<": 243, + ">>UNUSED_232<<": 244, + ">>UNUSED_233<<": 245, + ">>UNUSED_234<<": 246, + ">>UNUSED_235<<": 247, + ">>UNUSED_236<<": 248, + ">>UNUSED_237<<": 249, + ">>UNUSED_238<<": 250, + ">>UNUSED_239<<": 251, + ">>UNUSED_240<<": 252, + ">>UNUSED_241<<": 253, + ">>UNUSED_242<<": 254, + ">>UNUSED_243<<": 255, + ">>UNUSED_244<<": 256, + ">>UNUSED_245<<": 257, + ">>UNUSED_246<<": 258, + ">>UNUSED_247<<": 259, + ">>UNUSED_248<<": 260, + ">>UNUSED_249<<": 261, + ">>UNUSED_250<<": 262, + ">>UNUSED_251<<": 263, + ">>UNUSED_252<<": 264, + ">>UNUSED_253<<": 265, + ">>UNUSED_254<<": 266, + ">>UNUSED_255<<": 267, + ">>UNUSED_256<<": 268, + ">>UNUSED_257<<": 269, + ">>UNUSED_258<<": 270, + ">>UNUSED_259<<": 271, + ">>UNUSED_260<<": 272, + ">>UNUSED_261<<": 273, + ">>UNUSED_262<<": 274, + ">>UNUSED_263<<": 275, + ">>UNUSED_264<<": 276, + ">>UNUSED_265<<": 277, + ">>UNUSED_266<<": 278, + ">>UNUSED_267<<": 279, + ">>UNUSED_268<<": 280, + ">>UNUSED_269<<": 281, + ">>UNUSED_270<<": 282, + ">>UNUSED_271<<": 283, + ">>UNUSED_272<<": 284, + ">>UNUSED_273<<": 285, + ">>UNUSED_274<<": 286, + ">>UNUSED_275<<": 287, + ">>UNUSED_276<<": 288, + ">>UNUSED_277<<": 289, + ">>UNUSED_278<<": 290, + ">>UNUSED_279<<": 291, + ">>UNUSED_280<<": 292, + ">>UNUSED_281<<": 293, + ">>UNUSED_282<<": 294, + ">>UNUSED_283<<": 295, + ">>UNUSED_284<<": 296, + ">>UNUSED_285<<": 297, + ">>UNUSED_286<<": 298, + ">>UNUSED_287<<": 299, + ">>UNUSED_288<<": 300, + ">>UNUSED_289<<": 301, + ">>UNUSED_290<<": 302, + ">>UNUSED_291<<": 303, + ">>UNUSED_292<<": 304, + ">>UNUSED_293<<": 305, + ">>UNUSED_294<<": 306, + ">>UNUSED_295<<": 307, + ">>UNUSED_296<<": 308, + ">>UNUSED_297<<": 309, + ">>UNUSED_298<<": 310, + ">>UNUSED_299<<": 311, + ">>UNUSED_300<<": 312, + ">>UNUSED_301<<": 313, + ">>UNUSED_302<<": 314, + ">>UNUSED_303<<": 315, + ">>UNUSED_304<<": 316, + ">>UNUSED_305<<": 317, + ">>UNUSED_306<<": 318, + ">>UNUSED_307<<": 319, + ">>UNUSED_308<<": 320, + ">>UNUSED_309<<": 321, + ">>UNUSED_310<<": 322, + ">>UNUSED_311<<": 323, + ">>UNUSED_312<<": 324, + ">>UNUSED_313<<": 325, + ">>UNUSED_314<<": 326, + ">>UNUSED_315<<": 327, + ">>UNUSED_316<<": 328, + ">>UNUSED_317<<": 329, + ">>UNUSED_318<<": 330, + ">>UNUSED_319<<": 331, + ">>UNUSED_320<<": 332, + ">>UNUSED_321<<": 333, + ">>UNUSED_322<<": 334, + ">>UNUSED_323<<": 335, + ">>UNUSED_324<<": 336, + ">>UNUSED_325<<": 337, + ">>UNUSED_326<<": 338, + ">>UNUSED_327<<": 339, + ">>UNUSED_328<<": 340, + ">>UNUSED_329<<": 341, + ">>UNUSED_330<<": 342, + ">>UNUSED_331<<": 343, + ">>UNUSED_332<<": 344, + ">>UNUSED_333<<": 345, + ">>UNUSED_334<<": 346, + ">>UNUSED_335<<": 347, + ">>UNUSED_336<<": 348, + ">>UNUSED_337<<": 349, + ">>UNUSED_338<<": 350, + ">>UNUSED_339<<": 351, + ">>UNUSED_340<<": 352, + ">>UNUSED_341<<": 353, + ">>UNUSED_342<<": 354, + ">>UNUSED_343<<": 355, + ">>UNUSED_344<<": 356, + ">>UNUSED_345<<": 357, + ">>UNUSED_346<<": 358, + ">>UNUSED_347<<": 359, + ">>UNUSED_348<<": 360, + ">>UNUSED_349<<": 361, + ">>UNUSED_350<<": 362, + ">>UNUSED_351<<": 363, + ">>UNUSED_352<<": 364, + ">>UNUSED_353<<": 365, + ">>UNUSED_354<<": 366, + ">>UNUSED_355<<": 367, + ">>UNUSED_356<<": 368, + ">>UNUSED_357<<": 369, + ">>UNUSED_358<<": 370, + ">>UNUSED_359<<": 371, + ">>UNUSED_360<<": 372, + ">>UNUSED_361<<": 373, + ">>UNUSED_362<<": 374, + ">>UNUSED_363<<": 375, + ">>UNUSED_364<<": 376, + ">>UNUSED_365<<": 377, + ">>UNUSED_366<<": 378, + ">>UNUSED_367<<": 379, + ">>UNUSED_368<<": 380, + ">>UNUSED_369<<": 381, + ">>UNUSED_370<<": 382, + ">>UNUSED_371<<": 383, + ">>UNUSED_372<<": 384, + ">>UNUSED_373<<": 385, + ">>UNUSED_374<<": 386, + ">>UNUSED_375<<": 387, + ">>UNUSED_376<<": 388, + ">>UNUSED_377<<": 389, + ">>UNUSED_378<<": 390, + ">>UNUSED_379<<": 391, + ">>UNUSED_380<<": 392, + ">>UNUSED_381<<": 393, + ">>UNUSED_382<<": 394, + ">>UNUSED_383<<": 395, + ">>UNUSED_384<<": 396, + ">>UNUSED_385<<": 397, + ">>UNUSED_386<<": 398, + ">>UNUSED_387<<": 399, + ">>UNUSED_388<<": 400, + ">>UNUSED_389<<": 401, + ">>UNUSED_390<<": 402, + ">>UNUSED_391<<": 403, + ">>UNUSED_392<<": 404, + ">>UNUSED_393<<": 405, + ">>UNUSED_394<<": 406, + ">>UNUSED_395<<": 407, + ">>UNUSED_396<<": 408, + ">>UNUSED_397<<": 409, + ">>UNUSED_398<<": 410, + ">>UNUSED_399<<": 411, + ">>UNUSED_400<<": 412, + ">>UNUSED_401<<": 413, + ">>UNUSED_402<<": 414, + ">>UNUSED_403<<": 415, + ">>UNUSED_404<<": 416, + ">>UNUSED_405<<": 417, + ">>UNUSED_406<<": 418, + ">>UNUSED_407<<": 419, + ">>UNUSED_408<<": 420, + ">>UNUSED_409<<": 421, + ">>UNUSED_410<<": 422, + ">>UNUSED_411<<": 423, + ">>UNUSED_412<<": 424, + ">>UNUSED_413<<": 425, + ">>UNUSED_414<<": 426, + ">>UNUSED_415<<": 427, + ">>UNUSED_416<<": 428, + ">>UNUSED_417<<": 429, + ">>UNUSED_418<<": 430, + ">>UNUSED_419<<": 431, + ">>UNUSED_420<<": 432, + ">>UNUSED_421<<": 433, + ">>UNUSED_422<<": 434, + ">>UNUSED_423<<": 435, + ">>UNUSED_424<<": 436, + ">>UNUSED_425<<": 437, + ">>UNUSED_426<<": 438, + ">>UNUSED_427<<": 439, + ">>UNUSED_428<<": 440, + ">>UNUSED_429<<": 441, + ">>UNUSED_430<<": 442, + ">>UNUSED_431<<": 443, + ">>UNUSED_432<<": 444, + ">>UNUSED_433<<": 445, + ">>UNUSED_434<<": 446, + ">>UNUSED_435<<": 447, + ">>UNUSED_436<<": 448, + ">>UNUSED_437<<": 449, + ">>UNUSED_438<<": 450, + ">>UNUSED_439<<": 451, + ">>UNUSED_440<<": 452, + ">>UNUSED_441<<": 453, + ">>UNUSED_442<<": 454, + ">>UNUSED_443<<": 455, + ">>UNUSED_444<<": 456, + ">>UNUSED_445<<": 457, + ">>UNUSED_446<<": 458, + ">>UNUSED_447<<": 459, + ">>UNUSED_448<<": 460, + ">>UNUSED_449<<": 461, + ">>UNUSED_450<<": 462, + ">>UNUSED_451<<": 463, + ">>UNUSED_452<<": 464, + ">>UNUSED_453<<": 465, + ">>UNUSED_454<<": 466, + ">>UNUSED_455<<": 467, + ">>UNUSED_456<<": 468, + ">>UNUSED_457<<": 469, + ">>UNUSED_458<<": 470, + ">>UNUSED_459<<": 471, + ">>UNUSED_460<<": 472, + ">>UNUSED_461<<": 473, + ">>UNUSED_462<<": 474, + ">>UNUSED_463<<": 475, + ">>UNUSED_464<<": 476, + ">>UNUSED_465<<": 477, + ">>UNUSED_466<<": 478, + ">>UNUSED_467<<": 479, + ">>UNUSED_468<<": 480, + ">>UNUSED_469<<": 481, + ">>UNUSED_470<<": 482, + ">>UNUSED_471<<": 483, + ">>UNUSED_472<<": 484, + ">>UNUSED_473<<": 485, + ">>UNUSED_474<<": 486, + ">>UNUSED_475<<": 487, + ">>UNUSED_476<<": 488, + ">>UNUSED_477<<": 489, + ">>UNUSED_478<<": 490, + ">>UNUSED_479<<": 491, + ">>UNUSED_480<<": 492, + ">>UNUSED_481<<": 493, + ">>UNUSED_482<<": 494, + ">>UNUSED_483<<": 495, + ">>UNUSED_484<<": 496, + ">>UNUSED_485<<": 497, + ">>UNUSED_486<<": 498, + ">>UNUSED_487<<": 499, + ">>UNUSED_488<<": 500, + ">>UNUSED_489<<": 501, + ">>UNUSED_490<<": 502, + ">>UNUSED_491<<": 503, + ">>UNUSED_492<<": 504, + ">>UNUSED_493<<": 505, + ">>UNUSED_494<<": 506, + ">>UNUSED_495<<": 507, + ">>UNUSED_496<<": 508, + ">>UNUSED_497<<": 509, + ">>UNUSED_498<<": 510, + ">>UNUSED_499<<": 511, + ">>UNUSED_500<<": 512, + ">>UNUSED_501<<": 513, + ">>UNUSED_502<<": 514, + ">>UNUSED_503<<": 515, + ">>UNUSED_504<<": 516, + ">>UNUSED_505<<": 517, + ">>UNUSED_506<<": 518, + ">>UNUSED_507<<": 519, + ">>UNUSED_508<<": 520, + ">>UNUSED_509<<": 521, + ">>UNUSED_510<<": 522, + ">>UNUSED_511<<": 523, + "!": 524, + "\"": 525, + "#": 526, + "$": 527, + "%": 528, + "&": 529, + "'": 530, + "(": 531, + ")": 532, + "*": 533, + "+": 534, + ",": 535, + "-": 536, + ".": 537, + "/": 538, + "0": 539, + "1": 540, + "2": 541, + "3": 542, + "4": 543, + "5": 544, + "6": 545, + "7": 546, + "8": 547, + "9": 548, + ":": 549, + ";": 550, + "<": 551, + "=": 552, + ">": 553, + "?": 554, + "@": 555, + "A": 556, + "B": 557, + "C": 558, + "D": 559, + "E": 560, + "F": 561, + "G": 562, + "H": 563, + "I": 564, + "J": 565, + "K": 566, + "L": 567, + "M": 568, + "N": 569, + "O": 570, + "P": 571, + "Q": 572, + "R": 573, + "S": 574, + "T": 575, + "U": 576, + "V": 577, + "W": 578, + "X": 579, + "Y": 580, + "Z": 581, + "[": 582, + "\\": 583, + "]": 584, + "^": 585, + "_": 586, + "`": 587, + "a": 588, + "b": 589, + "c": 590, + "d": 591, + "e": 592, + "f": 593, + "g": 594, + "h": 595, + "i": 596, + "j": 597, + "k": 598, + "l": 599, + "m": 600, + "n": 601, + "o": 602, + "p": 603, + "q": 604, + "r": 605, + "s": 606, + "t": 607, + "u": 608, + "v": 609, + "w": 610, + "x": 611, + "y": 612, + "z": 613, + "{": 614, + "|": 615, + "}": 616, + "~": 617, + "¡": 618, + "¢": 619, + "£": 620, + "¤": 621, + "¥": 622, + "¦": 623, + "§": 624, + "¨": 625, + "©": 626, + "ª": 627, + "«": 628, + "¬": 629, + "®": 630, + "¯": 631, + "°": 632, + "±": 633, + "²": 634, + "³": 635, + "´": 636, + "µ": 637, + "¶": 638, + "·": 639, + "¸": 640, + "¹": 641, + "º": 642, + "»": 643, + "¼": 644, + "½": 645, + "¾": 646, + "¿": 647, + "Â": 648, + "Ã": 649, + "Ä": 650, + "Å": 651, + "Æ": 652, + "Ç": 653, + "È": 654, + "É": 655, + "Ê": 656, + "Ë": 657, + "Ì": 658, + "Í": 659, + "Î": 660, + "Ï": 661, + "Ð": 662, + "Ñ": 663, + "Ò": 664, + "Ó": 665, + "Ô": 666, + "Õ": 667, + "Ö": 668, + "×": 669, + "Ø": 670, + "Ù": 671, + "Ú": 672, + "Û": 673, + "Ü": 674, + "Ý": 675, + "Þ": 676, + "ß": 677, + "à": 678, + "á": 679, + "â": 680, + "ã": 681, + "ä": 682, + "å": 683, + "æ": 684, + "ç": 685, + "è": 686, + "é": 687, + "ê": 688, + "ë": 689, + "ì": 690, + "í": 691, + "î": 692, + "ï": 693, + "ð": 694, + "ñ": 695, + "ò": 696, + "ó": 697, + "ô": 698, + "Ā": 699, + "ā": 700, + "Ă": 701, + "ă": 702, + "Ą": 703, + "ą": 704, + "Ć": 705, + "ć": 706, + "Ĉ": 707, + "ĉ": 708, + "Ċ": 709, + "ċ": 710, + "Č": 711, + "č": 712, + "Ď": 713, + "ď": 714, + "Đ": 715, + "đ": 716, + "Ē": 717, + "ē": 718, + "Ĕ": 719, + "ĕ": 720, + "Ė": 721, + "ė": 722, + "Ę": 723, + "ę": 724, + "Ě": 725, + "ě": 726, + "Ĝ": 727, + "ĝ": 728, + "Ğ": 729, + "ğ": 730, + "Ġ": 731, + "ġ": 732, + "Ģ": 733, + "ģ": 734, + "Ĥ": 735, + "ĥ": 736, + "Ħ": 737, + "ħ": 738, + "Ĩ": 739, + "ĩ": 740, + "Ī": 741, + "ī": 742, + "Ĭ": 743, + "ĭ": 744, + "Į": 745, + "į": 746, + "İ": 747, + "ı": 748, + "IJ": 749, + "ij": 750, + "Ĵ": 751, + "ĵ": 752, + "Ķ": 753, + "ķ": 754, + "ĸ": 755, + "Ĺ": 756, + "ĺ": 757, + "Ļ": 758, + "ļ": 759, + "Ľ": 760, + "ľ": 761, + "Ŀ": 762, + "ŀ": 763, + "Ł": 764, + "ł": 765, + "Ń": 766, + "ĠĠ": 767, + "ĠĠĠĠ": 768, + "in": 769, + "Ġt": 770, + "er": 771, + "ĠĠĠĠĠĠĠĠ": 772, + "on": 773, + "Ġa": 774, + "re": 775, + "at": 776, + "he": 777, + "en": 778, + "or": 779, + "st": 780, + "al": 781, + "le": 782, + "ĠĠĠ": 783, + "Ġthe": 784, + "Ġc": 785, + "it": 786, + "--": 787, + "an": 788, + "ar": 789, + "Ġs": 790, + "es": 791, + "ion": 792, + "Ġf": 793, + "Ġo": 794, + "Ġp": 795, + "ing": 796, + "ro": 797, + "ed": 798, + "is": 799, + "ct": 800, + "Ġw": 801, + "Ġb": 802, + "ĉĉ": 803, + "nd": 804, + "ĊĊ": 805, + "ic": 806, + "et": 807, + "Ġm": 808, + "Ġin": 809, + "Ġd": 810, + "ou": 811, + "ent": 812, + "Ġof": 813, + "se": 814, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 815, + "čĊ": 816, + "Ġto": 817, + "id": 818, + "am": 819, + "----": 820, + "Ġn": 821, + "ol": 822, + "Ġand": 823, + "om": 824, + "ac": 825, + "Ġre": 826, + "ut": 827, + "Ġe": 828, + "im": 829, + "**": 830, + "as": 831, + "il": 832, + "ig": 833, + "lo": 834, + "FF": 835, + "ĠT": 836, + "ad": 837, + "if": 838, + "Ġh": 839, + "Ġth": 840, + "ur": 841, + "el": 842, + "==": 843, + "ot": 844, + "ĠS": 845, + "iv": 846, + "ĠĠĠĠĠĠĠ": 847, + "un": 848, + "ation": 849, + "od": 850, + "ag": 851, + "ĠC": 852, + "ĠĠĠĠĠ": 853, + "ĠA": 854, + "ĠI": 855, + "ul": 856, + "ate": 857, + "pe": 858, + "Ġl": 859, + "Ġis": 860, + "te": 861, + ");": 862, + "us": 863, + "th": 864, + "ce": 865, + "ue": 866, + "tr": 867, + "Ġg": 868, + "Ġv": 869, + "//": 870, + "Ġst": 871, + "um": 872, + "ab": 873, + "ch": 874, + "ay": 875, + "end": 876, + "em": 877, + "Ġfor": 878, + "ist": 879, + "--------": 880, + "ter": 881, + "ver": 882, + "##": 883, + "ht": 884, + "ly": 885, + "ap": 886, + "ir": 887, + "ction": 888, + "ame": 889, + "Ġbe": 890, + "ow": 891, + "Ġcon": 892, + "ĠP": 893, + "res": 894, + "op": 895, + "),": 896, + "ers": 897, + "int": 898, + "nt": 899, + "ve": 900, + "ex": 901, + "ef": 902, + "ith": 903, + "ĠD": 904, + "AA": 905, + "****": 906, + "ĉĉĉĉ": 907, + "ĠM": 908, + "ĠF": 909, + "Ġon": 910, + "ight": 911, + "pt": 912, + "=\"": 913, + "->": 914, + "qu": 915, + "::": 916, + "Ġy": 917, + "ine": 918, + "ĠR": 919, + "os": 920, + "ri": 921, + "Ġpro": 922, + "up": 923, + "Ġr": 924, + "ĠB": 925, + "age": 926, + "pl": 927, + "and": 928, + "px": 929, + "ort": 930, + "Ġex": 931, + "ub": 932, + "ĠE": 933, + "ess": 934, + "Ġthat": 935, + "ile": 936, + "con": 937, + "iz": 938, + "====": 939, + "ore": 940, + "ĠN": 941, + "ĠL": 942, + "ew": 943, + "ect": 944, + "ype": 945, + "ĠĠĠĠĠĠĠĠĠĠĠ": 946, + "ack": 947, + "Ġas": 948, + "ain": 949, + "urn": 950, + "Ġwith": 951, + "ata": 952, + "FFFF": 953, + "Ġan": 954, + "Ġwh": 955, + "able": 956, + "ff": 957, + "our": 958, + "ment": 959, + "ity": 960, + "Ġal": 961, + "))": 962, + "ass": 963, + "Ġor": 964, + "ke": 965, + "\",": 966, + "Ġif": 967, + "ult": 968, + "ĠW": 969, + "act": 970, + "oc": 971, + "orm": 972, + ").": 973, + "ure": 974, + "ive": 975, + "SS": 976, + "art": 977, + "und": 978, + "Ġit": 979, + "turn": 980, + "ib": 981, + "est": 982, + "ord": 983, + "ĠĠĠĠĠĠĠĠĠĠĠĠ": 984, + "rom": 985, + "ind": 986, + "ec": 987, + "alue": 988, + "ant": 989, + "ies": 990, + "port": 991, + "ĠG": 992, + "ans": 993, + "ud": 994, + "Ġcom": 995, + "Ġle": 996, + "DD": 997, + "Ġare": 998, + "all": 999, + "ere": 1000, + "ĠH": 1001, + "": 1074, + "ber": 1075, + "Ġcl": 1076, + "eg": 1077, + "ave": 1078, + "Ġsh": 1079, + "St": 1080, + "ace": 1081, + "Ġu": 1082, + "Ġx": 1083, + "com": 1084, + "ple": 1085, + "ust": 1086, + "rr": 1087, + "\">": 1088, + "ount": 1089, + "ost": 1090, + "}{": 1091, + "low": 1092, + "ĠĊ": 1093, + "ard": 1094, + "ss": 1095, + "Ġstr": 1096, + "Ġcan": 1097, + "ide": 1098, + "ice": 1099, + "ĉĉĉ": 1100, + "Ġse": 1101, + "__": 1102, + "ext": 1103, + "ical": 1104, + "Name": 1105, + "aram": 1106, + "one": 1107, + "========": 1108, + "frac": 1109, + "Ġk": 1110, + "ated": 1111, + "Ġreturn": 1112, + "\\(": 1113, + "ma": 1114, + "Id": 1115, + "ĠIn": 1116, + "Ġus": 1117, + "sp": 1118, + "ary": 1119, + "round": 1120, + "add": 1121, + "ĠV": 1122, + "ory": 1123, + "Ġlo": 1124, + "ug": 1125, + "ser": 1126, + "ach": 1127, + "uct": 1128, + "ON": 1129, + "},": 1130, + "ens": 1131, + "Ġim": 1132, + "Ġj": 1133, + "($": 1134, + "ance": 1135, + "ĠĠĠĠĠĠĠĠĠ": 1136, + "Con": 1137, + "Ġyou": 1138, + "Ġun": 1139, + "EN": 1140, + "ev": 1141, + "over": 1142, + "ations": 1143, + "ould": 1144, + "ents": 1145, + "sc": 1146, + "que": 1147, + "%%": 1148, + "Ġi": 1149, + "ence": 1150, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1151, + "pr": 1152, + "ak": 1153, + "we": 1154, + "Ġdo": 1155, + "ions": 1156, + "Ġhave": 1157, + "ĠTh": 1158, + "idth": 1159, + "ild": 1160, + "AL": 1161, + "ms": 1162, + "gr": 1163, + "Ġpr": 1164, + "row": 1165, + "ork": 1166, + "set": 1167, + "hen": 1168, + "čĊčĊ": 1169, + "ĠJ": 1170, + "ich": 1171, + "AAAA": 1172, + "AT": 1173, + "Ġerr": 1174, + "Ġcont": 1175, + "Type": 1176, + "AR": 1177, + "Ġfunction": 1178, + "Ġwas": 1179, + "ome": 1180, + "Ġclass": 1181, + "eng": 1182, + "Ġdis": 1183, + "Ġab": 1184, + "bject": 1185, + "ong": 1186, + "ell": 1187, + "get": 1188, + "\")": 1189, + "tring": 1190, + "Re": 1191, + "}.": 1192, + "div": 1193, + "fer": 1194, + "name": 1195, + "Ġpl": 1196, + "ail": 1197, + "Ġap": 1198, + "Ġthen": 1199, + "Ġres": 1200, + "cont": 1201, + "fo": 1202, + "Pro": 1203, + "ia": 1204, + "umber": 1205, + "ground": 1206, + "yst": 1207, + "return": 1208, + "Ġint": 1209, + "order": 1210, + "*/": 1211, + "ute": 1212, + "ther": 1213, + "DE": 1214, + "ual": 1215, + "line": 1216, + "sh": 1217, + "ST": 1218, + "clud": 1219, + "right": 1220, + "],": 1221, + "pec": 1222, + "OR": 1223, + "ge": 1224, + "ger": 1225, + "ll": 1226, + "ient": 1227, + "val": 1228, + "Ġvalue": 1229, + "enc": 1230, + "########": 1231, + "ear": 1232, + "Data": 1233, + "ackground": 1234, + "\\)": 1235, + "ĠSt": 1236, + "her": 1237, + "import": 1238, + "iel": 1239, + "rite": 1240, + "old": 1241, + "rt": 1242, + "The": 1243, + "ound": 1244, + "ous": 1245, + "own": 1246, + "ool": 1247, + "ule": 1248, + "lect": 1249, + "Ġsc": 1250, + "ript": 1251, + "Ġper": 1252, + "tern": 1253, + "ree": 1254, + "Ex": 1255, + "oint": 1256, + "\\[": 1257, + "color": 1258, + "ield": 1259, + "quest": 1260, + "ystem": 1261, + "\\]": 1262, + "FFFFFFFF": 1263, + "RE": 1264, + "aw": 1265, + "pos": 1266, + "lock": 1267, + "ax": 1268, + "Ġde": 1269, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1270, + "Ġhas": 1271, + "lic": 1272, + "ĉif": 1273, + "roup": 1274, + "Ġwhich": 1275, + "argin": 1276, + "Ġadd": 1277, + "type": 1278, + "Ġall": 1279, + "ph": 1280, + "Ġout": 1281, + "\";": 1282, + "ake": 1283, + "LE": 1284, + "def": 1285, + "Ġel": 1286, + "ĉreturn": 1287, + "data": 1288, + "ry": 1289, + "so": 1290, + "ener": 1291, + "ID": 1292, + "ert": 1293, + "Res": 1294, + "iew": 1295, + "Ġwill": 1296, + "test": 1297, + "IT": 1298, + "AC": 1299, + "ors": 1300, + "left": 1301, + "fig": 1302, + "ĠK": 1303, + "irst": 1304, + "min": 1305, + "ange": 1306, + "ix": 1307, + "SE": 1308, + "Ġtheir": 1309, + "Ġval": 1310, + "size": 1311, + "essage": 1312, + "ink": 1313, + "ne": 1314, + "ari": 1315, + "eter": 1316, + "Ġstring": 1317, + "String": 1318, + "ty": 1319, + "$\\": 1320, + "Ġset": 1321, + "unc": 1322, + "Ġget": 1323, + "sel": 1324, + "ater": 1325, + "--------------------------------": 1326, + "use": 1327, + "ull": 1328, + "Ġpos": 1329, + "****************": 1330, + "pon": 1331, + "IC": 1332, + "heck": 1333, + "box": 1334, + "alse": 1335, + "Ġpre": 1336, + "ource": 1337, + "mp": 1338, + "Value": 1339, + "Ġmy": 1340, + "Ġup": 1341, + "Ġone": 1342, + "/*": 1343, + "Ġsu": 1344, + "Ġne": 1345, + "eth": 1346, + "ced": 1347, + "Ġsub": 1348, + "Com": 1349, + "('": 1350, + "Ġro": 1351, + "key": 1352, + "ings": 1353, + "led": 1354, + "Ġsol": 1355, + "Ġover": 1356, + "Ġnew": 1357, + "Ġname": 1358, + "Ġsp": 1359, + "ates": 1360, + "âĢĻ": 1361, + "ble": 1362, + "oid": 1363, + "ative": 1364, + "Ġbut": 1365, + "Ġme": 1366, + "Ġelse": 1367, + "reate": 1368, + "ps": 1369, + "Ġli": 1370, + "Ġund": 1371, + "ength": 1372, + "rg": 1373, + "Ġso": 1374, + "Ġthey": 1375, + "Ġdata": 1376, + "trib": 1377, + "Ġtrans": 1378, + "Ġinter": 1379, + "rap": 1380, + "Ġtype": 1381, + "\":": 1382, + "play": 1383, + "ov": 1384, + "file": 1385, + "Ġyour": 1386, + "Ġman": 1387, + "Ġid": 1388, + "teger": 1389, + "ark": 1390, + "ose": 1391, + "Ġnumber": 1392, + "}$": 1393, + "ific": 1394, + "cept": 1395, + "Ġuse": 1396, + "clude": 1397, + "du": 1398, + ")*": 1399, + "Ġmore": 1400, + "Ch": 1401, + "sub": 1402, + "vent": 1403, + "Ġtime": 1404, + "ĠĠĠĠĠĠĠĠĠĠ": 1405, + "UT": 1406, + "List": 1407, + "li": 1408, + "efore": 1409, + "esc": 1410, + "pect": 1411, + "atch": 1412, + "Ġz": 1413, + "Ġwhen": 1414, + "erv": 1415, + "adding": 1416, + "les": 1417, + "ton": 1418, + "Ġpart": 1419, + "rit": 1420, + "ery": 1421, + "AB": 1422, + "not": 1423, + "AN": 1424, + "trans": 1425, + "================": 1426, + "DDDD": 1427, + "ader": 1428, + "cal": 1429, + "Ġcomm": 1430, + "log": 1431, + "Ġuint": 1432, + "io": 1433, + "Err": 1434, + "mod": 1435, + "To": 1436, + "Th": 1437, + "Ġlet": 1438, + "SSSS": 1439, + "WWWW": 1440, + "ys": 1441, + "AM": 1442, + "quire": 1443, + "urre": 1444, + "${": 1445, + "width": 1446, + "ublic": 1447, + "Ġen": 1448, + "ĠThis": 1449, + "ick": 1450, + "Ġstud": 1451, + "At": 1452, + "ays": 1453, + "ner": 1454, + "};": 1455, + "Ġnil": 1456, + "ĉĉĉĉĉĉĉĉ": 1457, + "ample": 1458, + "string": 1459, + "ues": 1460, + "wo": 1461, + "function": 1462, + "[]": 1463, + "col": 1464, + "self": 1465, + "((": 1466, + "ject": 1467, + "top": 1468, + "ator": 1469, + "ely": 1470, + "hed": 1471, + "atus": 1472, + "ody": 1473, + "enter": 1474, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1475, + "ott": 1476, + "ault": 1477, + "Set": 1478, + "ating": 1479, + "////": 1480, + "var": 1481, + "Path": 1482, + "Ġdef": 1483, + "ref": 1484, + "app": 1485, + "File": 1486, + "ob": 1487, + "ish": 1488, + "item": 1489, + "Cl": 1490, + "ym": 1491, + "af": 1492, + "AD": 1493, + "_{": 1494, + "lement": 1495, + "Get": 1496, + "imes": 1497, + "ng": 1498, + "read": 1499, + "ccess": 1500, + "list": 1501, + "Ġinto": 1502, + "Ġother": 1503, + "Ġrel": 1504, + "ĠY": 1505, + "abel": 1506, + "ED": 1507, + "tem": 1508, + "Ġunder": 1509, + "^{": 1510, + "sq": 1511, + "!=": 1512, + "ĠCh": 1513, + "):": 1514, + "Ġad": 1515, + "red": 1516, + "qual": 1517, + "Ġcol": 1518, + "Ġabout": 1519, + "ml": 1520, + "ĠCon": 1521, + "Ġmod": 1522, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1523, + "Ġform": 1524, + "ION": 1525, + "cess": 1526, + "tract": 1527, + "its": 1528, + "ET": 1529, + "Ġwork": 1530, + "Ġfile": 1531, + "loat": 1532, + "!!": 1533, + "ible": 1534, + "rray": 1535, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1536, + "nc": 1537, + "plic": 1538, + "Ġspec": 1539, + "son": 1540, + "RO": 1541, + "but": 1542, + "ied": 1543, + "Param": 1544, + "Ġconst": 1545, + "Ġte": 1546, + "\"\"": 1547, + "Ġthese": 1548, + "Ġind": 1549, + "iss": 1550, + "ics": 1551, + "of": 1552, + "CC": 1553, + "func": 1554, + "Ġonly": 1555, + "arn": 1556, + "eb": 1557, + "Add": 1558, + "arch": 1559, + "})": 1560, + "param": 1561, + "Ġbeg": 1562, + "$,": 1563, + "ĠPro": 1564, + "ify": 1565, + "ole": 1566, + "tp": 1567, + "date": 1568, + "inter": 1569, + "bt": 1570, + "Ġwere": 1571, + "EC": 1572, + "yle": 1573, + "dis": 1574, + "ey": 1575, + "Object": 1576, + "Ġthan": 1577, + "ottom": 1578, + "Int": 1579, + "Ġcall": 1580, + "ces": 1581, + "Ġam": 1582, + "ood": 1583, + "ie": 1584, + "inal": 1585, + "Ġmay": 1586, + "urrent": 1587, + "')": 1588, + "ock": 1589, + "Ġbackground": 1590, + "opt": 1591, + "md": 1592, + "rem": 1593, + "td": 1594, + "value": 1595, + "for": 1596, + "ense": 1597, + "Ġrec": 1598, + "Ġneed": 1599, + "Cont": 1600, + "ugh": 1601, + "ities": 1602, + "Ġalso": 1603, + "uild": 1604, + "Ġkn": 1605, + "ames": 1606, + "ages": 1607, + "rc": 1608, + "content": 1609, + "duct": 1610, + "Ġits": 1611, + "Ġany": 1612, + "rad": 1613, + "Ġcomp": 1614, + "Ġev": 1615, + "Ġthem": 1616, + "ian": 1617, + "Ġbo": 1618, + "Ġop": 1619, + "ternal": 1620, + "egin": 1621, + "Ġno": 1622, + "ĠUn": 1623, + "Ġfirst": 1624, + "Error": 1625, + "Ġbet": 1626, + "oth": 1627, + "iven": 1628, + "Ġtext": 1629, + "Ġexp": 1630, + "ash": 1631, + "cd": 1632, + "sum": 1633, + "ethod": 1634, + "Ġthere": 1635, + "yp": 1636, + "Ġsome": 1637, + "obj": 1638, + "Ġem": 1639, + "\");": 1640, + "Ġcal": 1641, + "igh": 1642, + "itle": 1643, + "Ġeach": 1644, + "olog": 1645, + "%;": 1646, + "the": 1647, + "tribute": 1648, + "$.": 1649, + "ĠRes": 1650, + "ĠGet": 1651, + "my": 1652, + "ĠIN": 1653, + "Ġlist": 1654, + "ra": 1655, + "Ġtrue": 1656, + "fl": 1657, + "Ġdif": 1658, + "Ġhow": 1659, + "der": 1660, + "ĠCom": 1661, + "Un": 1662, + "Ġresult": 1663, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1664, + "ility": 1665, + "Ġhis": 1666, + "++": 1667, + "by": 1668, + "arent": 1669, + "ask": 1670, + "face": 1671, + "ise": 1672, + "Ġtest": 1673, + "Ġwhere": 1674, + "Ġbegin": 1675, + "Ġmem": 1676, + "Ġfind": 1677, + "Ġcolor": 1678, + "rol": 1679, + "sqrt": 1680, + "AS": 1681, + "(-": 1682, + "opy": 1683, + "Ġreal": 1684, + "Ġmargin": 1685, + "atic": 1686, + "ĊĊĊ": 1687, + "ĠX": 1688, + "loc": 1689, + "irect": 1690, + "ices": 1691, + "ĠIt": 1692, + "cond": 1693, + "UL": 1694, + "].": 1695, + "Ġpe": 1696, + "input": 1697, + "cedure": 1698, + "ND": 1699, + "oug": 1700, + "ular": 1701, + "web": 1702, + "Ġtwo": 1703, + "rough": 1704, + "();": 1705, + "Ġrem": 1706, + "ns": 1707, + "$$": 1708, + "uth": 1709, + "ee": 1710, + "Ġshould": 1711, + "Ġsuch": 1712, + "Ġag": 1713, + "ctions": 1714, + "let": 1715, + "Ġreg": 1716, + "Ġpublic": 1717, + "Ġco": 1718, + "ider": 1719, + "Ġqu": 1720, + "Tr": 1721, + "cre": 1722, + "perty": 1723, + "Ġdec": 1724, + "Ġdist": 1725, + ",.": 1726, + "ution": 1727, + "max": 1728, + "begin": 1729, + "Ġline": 1730, + "])": 1731, + "ption": 1732, + "://": 1733, + "ause": 1734, + "################": 1735, + "ars": 1736, + "font": 1737, + "oun": 1738, + "Ġfont": 1739, + "uff": 1740, + "ES": 1741, + "ines": 1742, + "Ġsup": 1743, + "pp": 1744, + "];": 1745, + "idd": 1746, + "Ġborder": 1747, + "||": 1748, + "{$": 1749, + "Ġread": 1750, + "Ġyear": 1751, + "IS": 1752, + "Ġpoint": 1753, + "ature": 1754, + "ote": 1755, + "ons": 1756, + ")=": 1757, + "MP": 1758, + "Ġsim": 1759, + "FE": 1760, + "Ġwho": 1761, + "Ġph": 1762, + "Mod": 1763, + "ures": 1764, + "Ġact": 1765, + "ender": 1766, + "background": 1767, + "eta": 1768, + "be": 1769, + "Ġwidth": 1770, + "Ġprov": 1771, + "sol": 1772, + "Info": 1773, + "Ġused": 1774, + "print": 1775, + "lex": 1776, + "AP": 1777, + "Ġerror": 1778, + "ily": 1779, + "height": 1780, + "time": 1781, + "fter": 1782, + "border": 1783, + "new": 1784, + "margin": 1785, + "Ġright": 1786, + "En": 1787, + "Ind": 1788, + "Ġmin": 1789, + "ĠEx": 1790, + "ĠRe": 1791, + "ĉĉĉĉĉ": 1792, + "kit": 1793, + "work": 1794, + "UR": 1795, + "...": 1796, + "\"\\": 1797, + "ĠQ": 1798, + "Ġchar": 1799, + "ier": 1800, + "<<": 1801, + "ta": 1802, + "bar": 1803, + "}\\": 1804, + "LO": 1805, + "AG": 1806, + "CO": 1807, + "mer": 1808, + "Ġbeen": 1809, + "Ġacc": 1810, + "Ġour": 1811, + "ĠIf": 1812, + "ove": 1813, + "ined": 1814, + "load": 1815, + "void": 1816, + "ersion": 1817, + "pty": 1818, + "Ġaddress": 1819, + "``": 1820, + "important": 1821, + "ollow": 1822, + "arget": 1823, + "uint": 1824, + "als": 1825, + "Ġrequire": 1826, + "Ġposition": 1827, + "irc": 1828, + "reat": 1829, + "By": 1830, + "TE": 1831, + "ople": 1832, + "Ġĉ": 1833, + "arr": 1834, + "/>": 1835, + "pi": 1836, + "struct": 1837, + "erm": 1838, + "ade": 1839, + "ational": 1840, + "resent": 1841, + "hes": 1842, + "FFFFFFFFFFFFFFFF": 1843, + "âĢľ": 1844, + "([": 1845, + "ream": 1846, + "ween": 1847, + "ale": 1848, + "âĢĿ": 1849, + "bottom": 1850, + "Ġlike": 1851, + "ists": 1852, + "tes": 1853, + "const": 1854, + "ince": 1855, + "nav": 1856, + "br": 1857, + "Key": 1858, + "Ġthrough": 1859, + "Sc": 1860, + "Ġwould": 1861, + "Ġdiffer": 1862, + "ument": 1863, + "uf": 1864, + "ments": 1865, + "webkit": 1866, + "velo": 1867, + "';": 1868, + "Ġequ": 1869, + "lp": 1870, + "DB": 1871, + "Ġcontent": 1872, + "IF": 1873, + "Sh": 1874, + "ful": 1875, + "Ġvari": 1876, + "Se": 1877, + "Ġobject": 1878, + "ased": 1879, + "define": 1880, + "align": 1881, + "ases": 1882, + "\\<": 1883, + "Ġdiv": 1884, + "Ġusing": 1885, + "}}": 1886, + "ww": 1887, + "icon": 1888, + "Im": 1889, + "els": 1890, + "TR": 1891, + "Ġvar": 1892, + "AAAAAAAA": 1893, + ":#": 1894, + "Ġfalse": 1895, + "air": 1896, + "Ġtop": 1897, + "uc": 1898, + "///": 1899, + "ĠNew": 1900, + "ower": 1901, + "Time": 1902, + "formation": 1903, + "State": 1904, + "ĠTH": 1905, + "before": 1906, + "url": 1907, + "bl": 1908, + "ken": 1909, + "->{": 1910, + "IM": 1911, + "oken": 1912, + "vert": 1913, + "position": 1914, + "Ag": 1915, + "ĠTr": 1916, + "Ġâ": 1917, + "Ġcase": 1918, + "pre": 1919, + "map": 1920, + "ĠHe": 1921, + "orld": 1922, + "ward": 1923, + "&&": 1924, + "Ġfe": 1925, + "Ġdr": 1926, + "Ġhad": 1927, + "uring": 1928, + "FD": 1929, + "Ġwhat": 1930, + "ern": 1931, + "Ġmany": 1932, + "CH": 1933, + "sw": 1934, + "uto": 1935, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1936, + "|>": 1937, + "ponse": 1938, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1939, + "FB": 1940, + "Ġinteger": 1941, + "Ġfollow": 1942, + "index": 1943, + "ffect": 1944, + "Ġbetween": 1945, + "He": 1946, + "Ġever": 1947, + "tal": 1948, + "mt": 1949, + "bal": 1950, + "ann": 1951, + "ement": 1952, + "ict": 1953, + ".\"": 1954, + "serv": 1955, + "ector": 1956, + "sl": 1957, + "Ġpadding": 1958, + "Ġpeople": 1959, + "Ġdep": 1960, + "velop": 1961, + "MA": 1962, + "Config": 1963, + "Ġpol": 1964, + "Ġstart": 1965, + "ĠÐ": 1966, + "ential": 1967, + "ĠĠĊ": 1968, + "ious": 1969, + "ift": 1970, + "ENT": 1971, + "Ġmake": 1972, + "Ġinst": 1973, + "CE": 1974, + "ren": 1975, + "Ġleft": 1976, + "table": 1977, + "Ġnum": 1978, + "reg": 1979, + "Text": 1980, + "Request": 1981, + "enu": 1982, + "%>": 1983, + "Ġsm": 1984, + "search": 1985, + "Out": 1986, + "block": 1987, + "ply": 1988, + "cul": 1989, + "ited": 1990, + "Ġgr": 1991, + "bers": 1992, + "ains": 1993, + "ĠFor": 1994, + "ĠIF": 1995, + "ivate": 1996, + "btn": 1997, + "Test": 1998, + "System": 1999, + "ius": 2000, + "icense": 2001, + "Ġsystem": 2002, + "Ġstruct": 2003, + "ele": 2004, + "Ġgener": 2005, + "ocus": 2006, + "Index": 2007, + "eq": 2008, + "gram": 2009, + "alth": 2010, + "Item": 2011, + "ized": 2012, + "head": 2013, + "aint": 2014, + "Ġknow": 2015, + "OT": 2016, + "ervice": 2017, + "IG": 2018, + "CT": 2019, + "count": 2020, + "Ġmust": 2021, + "message": 2022, + "chool": 2023, + "atri": 2024, + "ilter": 2025, + "Ġfun": 2026, + "yn": 2027, + "error": 2028, + "raph": 2029, + "FC": 2030, + "indow": 2031, + "ba": 2032, + "Ġcor": 2033, + "Ġfl": 2034, + "ives": 2035, + "Ġsize": 2036, + "$(": 2037, + "Ġtotal": 2038, + "lib": 2039, + "*)": 2040, + "times": 2041, + "Ġstudents": 2042, + "Ġjust": 2043, + "include": 2044, + "Ġoff": 2045, + "angle": 2046, + "{\\": 2047, + "erver": 2048, + "med": 2049, + "info": 2050, + "UN": 2051, + "<%": 2052, + "script": 2053, + "Ġgo": 2054, + "Ġprint": 2055, + "Ġparam": 2056, + "ank": 2057, + "class": 2058, + "par": 2059, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 2060, + "bin": 2061, + "Ġstate": 2062, + "ackage": 2063, + ")+": 2064, + "Ġac": 2065, + "Ġcons": 2066, + "true": 2067, + "ainer": 2068, + "(_": 2069, + "group": 2070, + "atrix": 2071, + "escript": 2072, + "itial": 2073, + "ALL": 2074, + "fore": 2075, + "ting": 2076, + "Ġreturns": 2077, + "Ġkey": 2078, + "Token": 2079, + "Ġwrite": 2080, + "Ġheight": 2081, + "Ġq": 2082, + "FA": 2083, + "ĠSet": 2084, + "sg": 2085, + "sm": 2086, + ")^": 2087, + "label": 2088, + "Ġmax": 2089, + "\\\\": 2090, + "));": 2091, + "path": 2092, + "anc": 2093, + "padding": 2094, + "ural": 2095, + "less": 2096, + "For": 2097, + "Ġboth": 2098, + "Ġbec": 2099, + "lose": 2100, + "user": 2101, + "Ġmost": 2102, + "Ġed": 2103, + "EE": 2104, + "chn": 2105, + "Ġgiven": 2106, + "IP": 2107, + "amp": 2108, + "ush": 2109, + "********************************": 2110, + "(),": 2111, + "Ġnone": 2112, + "stand": 2113, + "Al": 2114, + "ĉc": 2115, + "body": 2116, + "Size": 2117, + "button": 2118, + "igned": 2119, + "#,": 2120, + "Ġass": 2121, + "Ġdisplay": 2122, + "%%%%": 2123, + "eters": 2124, + "Ġarg": 2125, + "word": 2126, + "Array": 2127, + "Or": 2128, + "mon": 2129, + "oci": 2130, + "wh": 2131, + "msg": 2132, + "vers": 2133, + "Ġchild": 2134, + "ants": 2135, + "Ġlong": 2136, + "Ġfound": 2137, + "hover": 2138, + "Ġdes": 2139, + "me": 2140, + "view": 2141, + "Ġwhile": 2142, + "Ġexample": 2143, + "address": 2144, + "co": 2145, + "ĠAl": 2146, + "....": 2147, + "ife": 2148, + "ified": 2149, + "cent": 2150, + "andle": 2151, + "dition": 2152, + "Ġdevelop": 2153, + "apt": 2154, + "IL": 2155, + "Ġsur": 2156, + "Ġdoes": 2157, + "Of": 2158, + "Ġsum": 2159, + "span": 2160, + "wn": 2161, + "ĠWh": 2162, + "*(": 2163, + "Ġafter": 2164, + "Val": 2165, + ",\"": 2166, + "New": 2167, + "Ġcomple": 2168, + "ten": 2169, + "ough": 2170, + "db": 2171, + "cdot": 2172, + "Message": 2173, + "Log": 2174, + "Ġpers": 2175, + "page": 2176, + "Ġbr": 2177, + "Context": 2178, + "Ġproble": 2179, + ",$": 2180, + "ets": 2181, + "uffer": 2182, + "ric": 2183, + "ts": 2184, + "link": 2185, + "Ġgroup": 2186, + "action": 2187, + "Up": 2188, + "Ġht": 2189, + "Ġher": 2190, + "ING": 2191, + "Ġlength": 2192, + "result": 2193, + "Ġimport": 2194, + "ware": 2195, + ":\"": 2196, + "olum": 2197, + "Ġevery": 2198, + "urs": 2199, + "Pr": 2200, + "())": 2201, + "back": 2202, + "state": 2203, + "ags": 2204, + "âĢĵ": 2205, + "xf": 2206, + "ĠWe": 2207, + "FFFFFF": 2208, + "////////": 2209, + "On": 2210, + "ism": 2211, + "rop": 2212, + "BC": 2213, + "transform": 2214, + "reak": 2215, + "iled": 2216, + "(*": 2217, + "ui": 2218, + "Form": 2219, + "Ġcenter": 2220, + "start": 2221, + "Field": 2222, + "(\\": 2223, + "Ġsign": 2224, + "Ġcould": 2225, + "<-": 2226, + "ever": 2227, + "OP": 2228, + "go": 2229, + "ache": 2230, + "ouble": 2231, + "Ġent": 2232, + "ession": 2233, + "ustom": 2234, + "ides": 2235, + "Ġappro": 2236, + "ateg": 2237, + "EX": 2238, + "try": 2239, + "ization": 2240, + "reen": 2241, + "');": 2242, + "day": 2243, + "vir": 2244, + "open": 2245, + "amily": 2246, + "ision": 2247, + "User": 2248, + "org": 2249, + "Ġcre": 2250, + "lear": 2251, + "Def": 2252, + "IR": 2253, + "ĉĉĉĉĉĉ": 2254, + "num": 2255, + "ĠString": 2256, + "DC": 2257, + "mit": 2258, + "Ġfact": 2259, + "Ġtri": 2260, + "Ġprocess": 2261, + "Count": 2262, + "ĠZ": 2263, + "ience": 2264, + "elf": 2265, + "Ġschool": 2266, + "html": 2267, + "Ġcurrent": 2268, + "dr": 2269, + "ression": 2270, + "(:": 2271, + "aining": 2272, + "Ġar": 2273, + "='": 2274, + "uch": 2275, + "Ġsolid": 2276, + "Ġinformation": 2277, + "ins": 2278, + "ormal": 2279, + "As": 2280, + "Ġsame": 2281, + "ology": 2282, + "Ġhelp": 2283, + "Ġeven": 2284, + "Ġdisc": 2285, + "mand": 2286, + "ically": 2287, + "dev": 2288, + "pan": 2289, + "Node": 2290, + "_,": 2291, + "Ġinclud": 2292, + "Result": 2293, + "ĠSh": 2294, + "Ġfield": 2295, + "other": 2296, + "etail": 2297, + "field": 2298, + "display": 2299, + "ences": 2300, + "ĠTo": 2301, + "CL": 2302, + "/**": 2303, + "code": 2304, + "Ġprocedure": 2305, + "vel": 2306, + "lobal": 2307, + "abled": 2308, + ":-": 2309, + "ĠAn": 2310, + "OD": 2311, + "EB": 2312, + "Ġdifferent": 2313, + "pha": 2314, + "\".": 2315, + "YP": 2316, + "config": 2317, + "Ġyears": 2318, + "point": 2319, + "Ġsee": 2320, + "ero": 2321, + "Create": 2322, + "sin": 2323, + "ptions": 2324, + "================================": 2325, + "Ġbl": 2326, + "lay": 2327, + "Color": 2328, + "Ġshow": 2329, + "roll": 2330, + "iver": 2331, + "check": 2332, + "Ġmethod": 2333, + "Ġlog": 2334, + "Response": 2335, + "Ġwrit": 2336, + "dir": 2337, + "dec": 2338, + "http": 2339, + "Ġinclude": 2340, + "angu": 2341, + "ning": 2342, + "Group": 2343, + "Ġpar": 2344, + "iter": 2345, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 2346, + "arrow": 2347, + "ape": 2348, + "ared": 2349, + "ister": 2350, + "json": 2351, + "Ġcheck": 2352, + "This": 2353, + "bb": 2354, + "aid": 2355, + "uration": 2356, + "title": 2357, + "View": 2358, + "ness": 2359, + "xc": 2360, + "lection": 2361, + "From": 2362, + "Ġresp": 2363, + "IB": 2364, + "Write": 2365, + "Ġdi": 2366, + "length": 2367, + "bol": 2368, + "Ġdirect": 2369, + "ĠSome": 2370, + "An": 2371, + "plication": 2372, + "this": 2373, + "olean": 2374, + "US": 2375, + "Str": 2376, + "xff": 2377, + "fa": 2378, + "TER": 2379, + "Ġrep": 2380, + "Ġvalues": 2381, + "math": 2382, + "header": 2383, + "pected": 2384, + "oon": 2385, + "active": 2386, + "QU": 2387, + "Ġworld": 2388, + "Ġinput": 2389, + "valid": 2390, + "Ġincre": 2391, + "Ġbecause": 2392, + "Ġbefore": 2393, + "\\).": 2394, + "cos": 2395, + "acter": 2396, + "ension": 2397, + "til": 2398, + "anag": 2399, + "+=": 2400, + "о": 2401, + "SC": 2402, + "!(": 2403, + "ĠInt": 2404, + "ually": 2405, + "Ġview": 2406, + "Ġsk": 2407, + "ĠLicense": 2408, + "hip": 2409, + "Ġtechn": 2410, + "Ġmon": 2411, + "select": 2412, + "Ġlocal": 2413, + "OL": 2414, + "><": 2415, + "Ġfr": 2416, + "ĠBo": 2417, + "nect": 2418, + "ats": 2419, + "ler": 2420, + "ypes": 2421, + "boxed": 2422, + "Ġart": 2423, + ".,": 2424, + "Ref": 2425, + "false": 2426, + "ivers": 2427, + "Status": 2428, + "itive": 2429, + "ĠResult": 2430, + "uthor": 2431, + "Ġblock": 2432, + "Ġexper": 2433, + "Parameter": 2434, + "Ġref": 2435, + "Ġamount": 2436, + "){": 2437, + "DA": 2438, + "Attribute": 2439, + "Bo": 2440, + "Ġlarg": 2441, + "otal": 2442, + "Ġoper": 2443, + "acity": 2444, + "Ġallow": 2445, + "ds": 2446, + "wner": 2447, + "Ġindex": 2448, + "Ġrun": 2449, + "Is": 2450, + ";}": 2451, + "ĠWrite": 2452, + "sert": 2453, + "Ġshe": 2454, + "Ġhigh": 2455, + "EL": 2456, + "ources": 2457, + "Qu": 2458, + "ingle": 2459, + "{\"": 2460, + "Ġinv": 2461, + "radius": 2462, + "ointer": 2463, + "а": 2464, + "Sub": 2465, + "no": 2466, + "Ġwell": 2467, + "ables": 2468, + "ML": 2469, + "ĠĠĠĠĊ": 2470, + "fe": 2471, + "utes": 2472, + "len": 2473, + "meric": 2474, + "viron": 2475, + "flow": 2476, + "ret": 2477, + "side": 2478, + "da": 2479, + "do": 2480, + "parent": 2481, + "Ġcondition": 2482, + "ember": 2483, + "Ab": 2484, + "Ġpass": 2485, + "Ġmed": 2486, + "lot": 2487, + "aking": 2488, + "Ġwater": 2489, + "rec": 2490, + "','": 2491, + "Code": 2492, + "ĠAs": 2493, + "Ġtem": 2494, + "rel": 2495, + "Loc": 2496, + "With": 2497, + "âĢĶ": 2498, + "pace": 2499, + "ĠÎ": 2500, + "Ġplan": 2501, + "ymbol": 2502, + ">>": 2503, + "ocument": 2504, + "older": 2505, + "Ġrange": 2506, + "AE": 2507, + "UB": 2508, + "call": 2509, + "iddle": 2510, + "Ġcontin": 2511, + "Ġposs": 2512, + "TH": 2513, + "net": 2514, + "ivity": 2515, + "Ġelement": 2516, + "Ġorg": 2517, + "Ġexpl": 2518, + "\\\"": 2519, + "ability": 2520, + "и": 2521, + "Ġoption": 2522, + "Ġsecond": 2523, + "Ġthose": 2524, + "stract": 2525, + "ribut": 2526, + "ULL": 2527, + "ency": 2528, + ")$": 2529, + "ĠHow": 2530, + "proto": 2531, + "down": 2532, + "image": 2533, + "Reg": 2534, + "xb": 2535, + "option": 2536, + "Ġback": 2537, + "Ġwithout": 2538, + "az": 2539, + "Ġnow": 2540, + "][": 2541, + "Ġmult": 2542, + "Ġnon": 2543, + "weight": 2544, + "ception": 2545, + "Ġext": 2546, + "----------------------------------------------------------------": 2547, + "Ġhum": 2548, + "\",\"": 2549, + "IO": 2550, + "Ġunderstand": 2551, + ",'": 2552, + "Ġcommun": 2553, + "book": 2554, + "е": 2555, + "Ġpartic": 2556, + "ĠOn": 2557, + "Address": 2558, + "gy": 2559, + "anguage": 2560, + "Dis": 2561, + "Wh": 2562, + "Ġprivate": 2563, + "Ġvoid": 2564, + "escription": 2565, + ":**": 2566, + "Ġduring": 2567, + "Ġextract": 2568, + "Not": 2569, + "ross": 2570, + "Ġlearn": 2571, + "ather": 2572, + "ux": 2573, + "come": 2574, + "itor": 2575, + "flex": 2576, + "BB": 2577, + "ished": 2578, + "Ġmat": 2579, + "Ġpat": 2580, + "off": 2581, + "Ġcoun": 2582, + "pc": 2583, + "inc": 2584, + "ĠAdd": 2585, + "Ġmap": 2586, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 2587, + ")/": 2588, + "arm": 2589, + "Ġhist": 2590, + "MS": 2591, + "Ad": 2592, + "PS": 2593, + "Ġrg": 2594, + "olv": 2595, + "rect": 2596, + "ĠEND": 2597, + "Ġway": 2598, + "../": 2599, + "Ġbeing": 2600, + "Ġcreate": 2601, + "Client": 2602, + "dep": 2603, + "aces": 2604, + "df": 2605, + "de": 2606, + "Ġmov": 2607, + "{}": 2608, + "ĠAmeric": 2609, + "Ġlast": 2610, + "Ġuser": 2611, + "odes": 2612, + "Map": 2613, + "XX": 2614, + "itional": 2615, + "Class": 2616, + "Ġhref": 2617, + "____": 2618, + "menu": 2619, + "UM": 2620, + "bs": 2621, + "Ġhealth": 2622, + "IV": 2623, + "imit": 2624, + "Ġbool": 2625, + "ands": 2626, + "akes": 2627, + "ots": 2628, + "Ġfollowing": 2629, + "vironment": 2630, + "Ġequation": 2631, + ">": 2849, + "nection": 2850, + "ron": 2851, + "Version": 2852, + "CM": 2853, + "mat": 2854, + "AF": 2855, + "**:": 2856, + "Var": 2857, + "ffff": 2858, + ")]": 2859, + ">=": 2860, + "gg": 2861, + "KE": 2862, + "Ġevent": 2863, + "ored": 2864, + "clus": 2865, + "ique": 2866, + "ference": 2867, + "Ġoverride": 2868, + "}$.": 2869, + "procedure": 2870, + "lish": 2871, + "Ġser": 2872, + "\"),": 2873, + "Ġauto": 2874, + "Op": 2875, + "ait": 2876, + "cp": 2877, + "Ġsaid": 2878, + "static": 2879, + "char": 2880, + "ĉĉĉĉĉĉĉ": 2881, + "Ġsrc": 2882, + "Ġchildren": 2883, + "Ġarray": 2884, + "ftware": 2885, + "Ġpresent": 2886, + "none": 2887, + "Ġocc": 2888, + "ium": 2889, + "Ġrequired": 2890, + "Ġcomput": 2891, + "CB": 2892, + "[\"": 2893, + "ulation": 2894, + "utine": 2895, + "Ġcirc": 2896, + "attribute": 2897, + "ponent": 2898, + "xd": 2899, + "Ġnull": 2900, + "xa": 2901, + "Table": 2902, + "Ġlife": 2903, + "Lo": 2904, + "fff": 2905, + "We": 2906, + "ively": 2907, + "ibr": 2908, + "ted": 2909, + "last": 2910, + "elete": 2911, + "Ġenc": 2912, + "DF": 2913, + "Ġmark": 2914, + "CA": 2915, + "github": 2916, + "Ġequal": 2917, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 2918, + "ither": 2919, + ")),": 2920, + "Ġob": 2921, + "xx": 2922, + "Acc": 2923, + "Ġnumbers": 2924, + "End": 2925, + "Ġbody": 2926, + "oration": 2927, + "roid": 2928, + "Ġwant": 2929, + "Fr": 2930, + "Am": 2931, + "plit": 2932, + "If": 2933, + "eff": 2934, + "Host": 2935, + "OUT": 2936, + "img": 2937, + "Ġdefault": 2938, + "status": 2939, + "Ġfloat": 2940, + "Col": 2941, + "Ġtake": 2942, + "ĠIm": 2943, + "Ġhere": 2944, + "App": 2945, + "ĠOF": 2946, + "Ġproduct": 2947, + "aster": 2948, + "ained": 2949, + "fun": 2950, + "ĊĠĠĠĠĊ": 2951, + "version": 2952, + "ĉĉĉĉĉĉĉĉĉ": 2953, + "ĠBut": 2954, + "ĠSo": 2955, + "AME": 2956, + "ogma": 2957, + "Ġmet": 2958, + ")).": 2959, + "ped": 2960, + "token": 2961, + "quence": 2962, + "Ġest": 2963, + "EM": 2964, + "ru": 2965, + "################################": 2966, + "perties": 2967, + "н": 2968, + "earch": 2969, + "Ġprote": 2970, + "Ġspe": 2971, + "aus": 2972, + "shadow": 2973, + "opyright": 2974, + "Ġdid": 2975, + "init": 2976, + "ĠLet": 2977, + ")))": 2978, + "etwork": 2979, + "ĠInteger": 2980, + "Ġdev": 2981, + "mb": 2982, + "src": 2983, + "Ġbel": 2984, + "ĠThey": 2985, + "Ġstop": 2986, + "array": 2987, + "\"/": 2988, + "ixed": 2989, + "tim": 2990, + "BA": 2991, + "ĠCON": 2992, + "Ġfore": 2993, + "$_": 2994, + "Ġrgba": 2995, + "WWWWWWWW": 2996, + "urity": 2997, + "âĤ": 2998, + "AW": 2999, + "ĠErr": 3000, + "rior": 3001, + "ptr": 3002, + "stit": 3003, + "Ðķ": 3004, + "Ġide": 3005, + "FI": 3006, + "float": 3007, + "part": 3008, + "Ġmemory": 3009, + "Ġhttp": 3010, + "SSSSSSSS": 3011, + "center": 3012, + "Ġdim": 3013, + "irt": 3014, + "output": 3015, + "ĠOR": 3016, + "ories": 3017, + "Pl": 3018, + "Ġmight": 3019, + "atory": 3020, + "andler": 3021, + "ately": 3022, + "Ġbook": 3023, + "Ġinitial": 3024, + "bf": 3025, + "Ġtoken": 3026, + "\").": 3027, + "grid": 3028, + "ending": 3029, + "exp": 3030, + "Equal": 3031, + "peat": 3032, + "Ġgrow": 3033, + "inition": 3034, + "Ġpoints": 3035, + "Start": 3036, + "Ġexternal": 3037, + "ĠSystem": 3038, + "request": 3039, + "Ġchange": 3040, + "Ġaround": 3041, + "Ġok": 3042, + "uments": 3043, + "Number": 3044, + "Ġoutput": 3045, + "ctor": 3046, + "Ġmiddle": 3047, + "inary": 3048, + "OS": 3049, + "Check": 3050, + "imal": 3051, + ",:": 3052, + "object": 3053, + "module": 3054, + "bg": 3055, + "sec": 3056, + "has": 3057, + "ĠAt": 3058, + "ought": 3059, + "tings": 3060, + "www": 3061, + "Us": 3062, + "EF": 3063, + "yc": 3064, + "ÑĢ": 3065, + "ĠSe": 3066, + "Ġcost": 3067, + "afe": 3068, + "CON": 3069, + "Ġchang": 3070, + "man": 3071, + "Ġloc": 3072, + "write": 3073, + "Manag": 3074, + "Input": 3075, + "model": 3076, + "iving": 3077, + "OF": 3078, + "ĠValue": 3079, + "Ġansw": 3080, + ")-": 3081, + "Ġmember": 3082, + "assert": 3083, + "Ġrece": 3084, + "LL": 3085, + "ĠMoon": 3086, + "lev": 3087, + "Ġintent": 3088, + "Ġtarget": 3089, + "ends": 3090, + "ior": 3091, + "post": 3092, + "Image": 3093, + "Model": 3094, + "Ġcount": 3095, + "Dr": 3096, + "Vector": 3097, + "section": 3098, + "Ġself": 3099, + "\">>": 3100, + "los": 3101, + "gress": 3102, + "Ġtable": 3103, + "Ġpract": 3104, + "vailable": 3105, + "Ġstudy": 3106, + "All": 3107, + "req": 3108, + "Ġmessage": 3109, + "with": 3110, + "Window": 3111, + "ident": 3112, + "ters": 3113, + "uck": 3114, + "Ġcover": 3115, + "Ġpossible": 3116, + "ally": 3117, + "ĠPr": 3118, + "Ġrad": 3119, + "rid": 3120, + "ugg": 3121, + "INE": 3122, + "Resource": 3123, + "media": 3124, + "alle": 3125, + "<<\"": 3126, + "rib": 3127, + "Ġfocus": 3128, + "circ": 3129, + "atab": 3130, + "Ġurl": 3131, + "icle": 3132, + "lean": 3133, + "abs": 3134, + "Ġsuccess": 3135, + "local": 3136, + "Ñģ": 3137, + "control": 3138, + "mine": 3139, + "Ġdistribut": 3140, + "Ġadv": 3141, + "ĠčĊ": 3142, + "Ġincluding": 3143, + "Ġcert": 3144, + "sim": 3145, + "Ġspecific": 3146, + "ĉĊ": 3147, + "VER": 3148, + "land": 3149, + "UP": 3150, + "ours": 3151, + "ĠComp": 3152, + "first": 3153, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 3154, + "ogle": 3155, + "Fl": 3156, + "main": 3157, + "dp": 3158, + "Ġindiv": 3159, + "Property": 3160, + "ls": 3161, + "Ġke": 3162, + "olution": 3163, + "context": 3164, + "BE": 3165, + "Length": 3166, + "prov": 3167, + "ĠNULL": 3168, + "AY": 3169, + "Ġconf": 3170, + "NO": 3171, + "iff": 3172, + "Ġcontext": 3173, + "Ġold": 3174, + "Source": 3175, + "ĠAr": 3176, + "focus": 3177, + "Call": 3178, + "Action": 3179, + "ead": 3180, + "pm": 3181, + "ĉfor": 3182, + "PI": 3183, + "su": 3184, + "Ġsince": 3185, + "ĠDogma": 3186, + "Ġcontrol": 3187, + "Ġweb": 3188, + "SI": 3189, + "({": 3190, + "iversity": 3191, + "asc": 3192, + "Ġfinal": 3193, + "Ġcap": 3194, + "oto": 3195, + "Ġmean": 3196, + "Ġredu": 3197, + "ways": 3198, + "aging": 3199, + "Ġorgan": 3200, + "Content": 3201, + "ĠFFFF": 3202, + "vest": 3203, + "Ġgood": 3204, + "ĠThese": 3205, + "$\"": 3206, + "Ġplace": 3207, + "Ġtitle": 3208, + "Ġsource": 3209, + "Ġarr": 3210, + ")}": 3211, + "```": 3212, + "Ġet": 3213, + "kn": 3214, + "Ġoften": 3215, + "Ġimportant": 3216, + "tx": 3217, + "omen": 3218, + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF": 3219, + "Ġproject": 3220, + "Ġreport": 3221, + "Ġlanguage": 3222, + "Ġexist": 3223, + "Ġsw": 3224, + "Ġterms": 3225, + "ven": 3226, + "Ġhuman": 3227, + ">.": 3228, + "Ġmodel": 3229, + "alk": 3230, + "ASS": 3231, + "Ġcare": 3232, + "Ġbox": 3233, + "Ġanal": 3234, + "quare": 3235, + "(@": 3236, + "orth": 3237, + "ĠIS": 3238, + "Ġbas": 3239, + "ĠPl": 3240, + "Read": 3241, + "orn": 3242, + "ilar": 3243, + "Dir": 3244, + "ĠData": 3245, + "Ġconfig": 3246, + "atural": 3247, + "sender": 3248, + "WWW": 3249, + "Per": 3250, + "ients": 3251, + "Now": 3252, + "Ver": 3253, + "andard": 3254, + "cb": 3255, + "olume": 3256, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 3257, + "Lex": 3258, + "izing": 3259, + "ĠList": 3260, + "Ġbus": 3261, + "Stream": 3262, + "Ġfa": 3263, + "Ġmatch": 3264, + "Command": 3265, + "Ġanother": 3266, + "Ġbreak": 3267, + "Ġindividual": 3268, + "run": 3269, + "quad": 3270, + "Ġdon": 3271, + "Ġcharacter": 3272, + "Block": 3273, + "ently": 3274, + "Ap": 3275, + "Ġbytes": 3276, + "endif": 3277, + "Ġselect": 3278, + "Ġfood": 3279, + "ĠTHE": 3280, + "ording": 3281, + "ä¸": 3282, + "Ġhim": 3283, + "css": 3284, + "Ġcult": 3285, + "null": 3286, + "Ptr": 3287, + "ebug": 3288, + "}}{": 3289, + "Ġsubject": 3290, + "(!": 3291, + "ane": 3292, + "tra": 3293, + "]$": 3294, + "Ġpres": 3295, + "lg": 3296, + "Char": 3297, + "etails": 3298, + "unit": 3299, + "%%%%%%%%": 3300, + "card": 3301, + "annel": 3302, + "ĠST": 3303, + "ĠFr": 3304, + "}/": 3305, + "Ġsever": 3306, + "ĠAnd": 3307, + "PL": 3308, + "SQ": 3309, + "Ġlaw": 3310, + "ĉvar": 3311, + "ids": 3312, + "Ġinternal": 3313, + "andom": 3314, + "-\\": 3315, + "<=": 3316, + "ĠAp": 3317, + "apter": 3318, + "ury": 3319, + "Ġcourse": 3320, + "else": 3321, + "Point": 3322, + "Ġversion": 3323, + "soft": 3324, + "Ġconstant": 3325, + "(&": 3326, + "ĠBoolean": 3327, + "temp": 3328, + "Ġaccount": 3329, + "So": 3330, + "inding": 3331, + "nder": 3332, + "Ġbeh": 3333, + "signed": 3334, + "Ġrest": 3335, + "bool": 3336, + "And": 3337, + "oh": 3338, + "Ġinterest": 3339, + "imary": 3340, + "bit": 3341, + "Ġthink": 3342, + "GL": 3343, + "iple": 3344, + "routine": 3345, + "Ġmain": 3346, + "ĠCal": 3347, + "ĠAll": 3348, + "ibility": 3349, + "inst": 3350, + "lick": 3351, + "Ġbase": 3352, + "Ass": 3353, + "Ġknown": 3354, + "ync": 3355, + "sk": 3356, + "Ġformat": 3357, + "transition": 3358, + "Ġdig": 3359, + "byte": 3360, + "space": 3361, + "family": 3362, + "Ġempty": 3363, + "names": 3364, + "omic": 3365, + "PRO": 3366, + "CF": 3367, + "PAR": 3368, + "Run": 3369, + "alf": 3370, + "Ġintro": 3371, + "Ġdefined": 3372, + "-->": 3373, + "ention": 3374, + "Ġvalid": 3375, + "lin": 3376, + "Ġtre": 3377, + "ĠTHEN": 3378, + "Ġmeans": 3379, + "ready": 3380, + "resh": 3381, + "Lexer": 3382, + "ek": 3383, + "DDD": 3384, + "cer": 3385, + "SH": 3386, + "cm": 3387, + "irection": 3388, + "Ġbased": 3389, + "Ġbro": 3390, + "aving": 3391, + "<": 3577, + "Ġmaterial": 3578, + "mark": 3579, + "sv": 3580, + "TEGER": 3581, + "acy": 3582, + "drop": 3583, + "ÐĺÐ": 3584, + "Label": 3585, + "iness": 3586, + "ĉt": 3587, + "Ġtreat": 3588, + "Ġfin": 3589, + "Ġcur": 3590, + "oo": 3591, + "ĉmy": 3592, + "Right": 3593, + "////////////////": 3594, + "ÐķÐ": 3595, + "ĠPh": 3596, + "dim": 3597, + "pose": 3598, + "Ġins": 3599, + "Num": 3600, + "icrosoft": 3601, + "host": 3602, + "ves": 3603, + "doc": 3604, + "deg": 3605, + "ĠCont": 3606, + "tn": 3607, + "Buffer": 3608, + "ql": 3609, + "Ġaut": 3610, + "cf": 3611, + "Tem": 3612, + "ĠSince": 3613, + "tag": 3614, + "current": 3615, + "Ġresults": 3616, + "Ġeduc": 3617, + "utions": 3618, + "Ġlarge": 3619, + "Box": 3620, + "iles": 3621, + "Ġapply": 3622, + "Ġinterface": 3623, + "Ġlevel": 3624, + "Ġleast": 3625, + "Ġprom": 3626, + "ĠTO": 3627, + "moz": 3628, + "ĠFF": 3629, + "atures": 3630, + "Null": 3631, + "ĠEng": 3632, + "TO": 3633, + "Valid": 3634, + "Ġintegr": 3635, + "foot": 3636, + "por": 3637, + "omet": 3638, + "Be": 3639, + "Ġlight": 3640, + "VAL": 3641, + "apping": 3642, + "Order": 3643, + "Ġens": 3644, + "=$": 3645, + "typ": 3646, + "EA": 3647, + "gether": 3648, + "ocol": 3649, + "Ġlink": 3650, + "outh": 3651, + "Ġhistory": 3652, + "Ġfree": 3653, + "hing": 3654, + "Ġperiod": 3655, + "filter": 3656, + "Ġiss": 3657, + "Ġoriginal": 3658, + "kg": 3659, + "di": 3660, + "ration": 3661, + "GR": 3662, + "]);": 3663, + "clusion": 3664, + "ĠDis": 3665, + "ATION": 3666, + "Ġfour": 3667, + "It": 3668, + "mem": 3669, + "Ġendif": 3670, + "Ġamong": 3671, + "Ġphys": 3672, + "STR": 3673, + "Ġang": 3674, + "Em": 3675, + "update": 3676, + "plied": 3677, + "Ġgre": 3678, + "alloc": 3679, + "agn": 3680, + "fmt": 3681, + "Ġstatus": 3682, + "just": 3683, + "Msg": 3684, + "asing": 3685, + "Ġgu": 3686, + "env": 3687, + "ysis": 3688, + "mary": 3689, + "std": 3690, + "^\\": 3691, + "Ġnode": 3692, + "MO": 3693, + "grad": 3694, + "ication": 3695, + "Ġhead": 3696, + "Ġdays": 3697, + "Enc": 3698, + "Ġelect": 3699, + "Ġunderstanding": 3700, + "lood": 3701, + "Ġassoci": 3702, + "Ġwhite": 3703, + "Options": 3704, + "ĠCreate": 3705, + "Ġtoo": 3706, + "Ġsolve": 3707, + "').": 3708, + "icult": 3709, + "ecess": 3710, + "ho": 3711, + "urch": 3712, + "ULT": 3713, + "PR": 3714, + "Exception": 3715, + "acc": 3716, + "https": 3717, + "Ġexpress": 3718, + "FO": 3719, + "Ġstd": 3720, + "Ġemp": 3721, + "ram": 3722, + "bytes": 3723, + "ĠOut": 3724, + "acing": 3725, + "MT": 3726, + "Instance": 3727, + "equ": 3728, + "create": 3729, + "ĠUniversity": 3730, + "Ġparameter": 3731, + "Ġprior": 3732, + "Ġzero": 3733, + "Ġclear": 3734, + "core": 3735, + "ĠId": 3736, + "Ġclose": 3737, + "Ġteach": 3738, + "estamp": 3739, + "ired": 3740, + "........": 3741, + "Ġpay": 3742, + "oy": 3743, + "ething": 3744, + "ĠAmerican": 3745, + "ategory": 3746, + "Sup": 3747, + "client": 3748, + "ips": 3749, + "MAX": 3750, + "Ġstory": 3751, + "+\\": 3752, + "ills": 3753, + "My": 3754, + "product": 3755, + "ĠOr": 3756, + "<>": 3757, + "dat": 3758, + "urther": 3759, + "Params": 3760, + "target": 3761, + "ental": 3762, + "Header": 3763, + "ians": 3764, + "EQ": 3765, + "cell": 3766, + "ĠWhat": 3767, + "Ġsignific": 3768, + "ĠĊĊ": 3769, + "Ġsym": 3770, + "});": 3771, + ")\"": 3772, + "Ident": 3773, + "pack": 3774, + "plicit": 3775, + "CS": 3776, + "Ġtextbook": 3777, + "TYPE": 3778, + "Ġcertain": 3779, + "Ġeq": 3780, + "ges": 3781, + "istic": 3782, + "(--": 3783, + "Ġeas": 3784, + "[:": 3785, + "ĠAcc": 3786, + "ariable": 3787, + "ĠThere": 3788, + ")\\": 3789, + "uy": 3790, + "Ġtogether": 3791, + "Ġinvolv": 3792, + "oper": 3793, + "ĠHowever": 3794, + "ai": 3795, + "straint": 3796, + "ilt": 3797, + "make": 3798, + "Ġfew": 3799, + "Ġdest": 3800, + "Ġagainst": 3801, + "Ġdirectly": 3802, + "ãģ": 3803, + "known": 3804, + "Ðŀ": 3805, + "Ġlabel": 3806, + "Ġwriting": 3807, + "aut": 3808, + "Rep": 3809, + "NAME": 3810, + "Abstract": 3811, + "ula": 3812, + "Ġinf": 3813, + "Ġdise": 3814, + "verage": 3815, + "ches": 3816, + "}$,": 3817, + "Max": 3818, + "ci": 3819, + "Ġfri": 3820, + "Ġsocial": 3821, + "Ġenergy": 3822, + "Pre": 3823, + "Mode": 3824, + "ĉb": 3825, + "Ġalong": 3826, + "ral": 3827, + "Ġalloc": 3828, + "Ġrot": 3829, + "Ġprovide": 3830, + "Ġothers": 3831, + "fs": 3832, + "Let": 3833, + "lim": 3834, + "SO": 3835, + "ordin": 3836, + "Ġplanet": 3837, + "oding": 3838, + "olve": 3839, + "Ġtypes": 3840, + "uture": 3841, + "]),": 3842, + "Hash": 3843, + "oud": 3844, + "idget": 3845, + "Ġcrit": 3846, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 3847, + "Ġkeep": 3848, + "Ġden": 3849, + "ĠType": 3850, + "Ġparent": 3851, + "\\(\\": 3852, + "IND": 3853, + "Ġknowledge": 3854, + "Click": 3855, + "thing": 3856, + "Ġant": 3857, + "Gener": 3858, + "Dep": 3859, + "Ġrecord": 3860, + "Ġpositive": 3861, + "Ġvarious": 3862, + "Ġrelative": 3863, + "pb": 3864, + "number": 3865, + "Ġput": 3866, + "Ġbetter": 3867, + "Ġbi": 3868, + "change": 3869, + "Ġsat": 3870, + "ĠReg": 3871, + "thon": 3872, + "ATA": 3873, + "Ġalready": 3874, + "Control": 3875, + "hash": 3876, + "=~": 3877, + "PC": 3878, + "Format": 3879, + "Ġparticular": 3880, + "valu": 3881, + ".âĢĿ": 3882, + "Ġtrad": 3883, + "Ġdesign": 3884, + "sup": 3885, + "step": 3886, + "Ġuntil": 3887, + "DIR": 3888, + "ZE": 3889, + "Ġlit": 3890, + "Ġhapp": 3891, + "BF": 3892, + "aption": 3893, + "ae": 3894, + "Ġlimit": 3895, + "SET": 3896, + "board": 3897, + "ws": 3898, + "xs": 3899, + "iev": 3900, + "DEF": 3901, + "Ġpub": 3902, + "eline": 3903, + "disabled": 3904, + "Ġalways": 3905, + "\"))": 3906, + "{-": 3907, + "Ġaction": 3908, + "Ġnatural": 3909, + ".*": 3910, + "Ġturn": 3911, + "Manager": 3912, + "rypt": 3913, + "Hel": 3914, + "Ġsingle": 3915, + "Ġfre": 3916, + "SON": 3917, + "png": 3918, + "ibrary": 3919, + "expected": 3920, + "event": 3921, + "arning": 3922, + "Ġlay": 3923, + "ĠDr": 3924, + "[$": 3925, + "ze": 3926, + "adata": 3927, + "Mem": 3928, + "lt": 3929, + "ĠCan": 3930, + "ĠCheck": 3931, + "Ġabsolute": 3932, + "tributes": 3933, + "aced": 3934, + ")%": 3935, + "<:": 3936, + "Ġred": 3937, + "Ġben": 3938, + "rightarrow": 3939, + "Ġsubroutine": 3940, + "DS": 3941, + "ĠGener": 3942, + "icient": 3943, + "Ġdevelopment": 3944, + "ajor": 3945, + "real": 3946, + "Ġspecial": 3947, + "Ġport": 3948, + "Ġmonth": 3949, + "comm": 3950, + "ump": 3951, + "RC": 3952, + "Owner": 3953, + "Ġthought": 3954, + "arge": 3955, + "source": 3956, + "Ġfail": 3957, + "Ġquestion": 3958, + "rist": 3959, + "inner": 3960, + "Ġur": 3961, + "Cal": 3962, + "Ġrespon": 3963, + "interface": 3964, + "ĠTop": 3965, + "ĠTrans": 3966, + "ĠFile": 3967, + "evel": 3968, + "yy": 3969, + "ĠINTEGER": 3970, + "AGE": 3971, + "Require": 3972, + "wards": 3973, + "Ġsay": 3974, + "Ġrate": 3975, + "tool": 3976, + "\"}.": 3977, + "att": 3978, + "SION": 3979, + "Ġbottom": 3980, + "ological": 3981, + "GE": 3982, + "ĠSub": 3983, + "Ġimplement": 3984, + "Ġsomething": 3985, + "Ġfunc": 3986, + "Ġchalleng": 3987, + "ights": 3988, + "Ġyoung": 3989, + "Ġrole": 3990, + "Ġnames": 3991, + "Ġbest": 3992, + "Ġsequence": 3993, + "Ġmodule": 3994, + "Do": 3995, + "%,": 3996, + "Dev": 3997, + "iting": 3998, + "Ġseveral": 3999, + "ĠUse": 4000, + "irectory": 4001, + "Ther": 4002, + "[|": 4003, + "Ġfunctions": 4004, + "VM": 4005, + "Ġmsg": 4006, + "gl": 4007, + "Ġobt": 4008, + "ii": 4009, + "AX": 4010, + "Ġelements": 4011, + "Ġsolution": 4012, + "Ġge": 4013, + "ORM": 4014, + "export": 4015, + "amed": 4016, + "ĠCompiler": 4017, + "Ġdue": 4018, + "Oper": 4019, + "ĠUp": 4020, + "tmp": 4021, + "Ġage": 4022, + "Ġalign": 4023, + "Ġgreat": 4024, + "ĠInter": 4025, + "params": 4026, + "Ġbecome": 4027, + "Filter": 4028, + "fix": 4029, + "'(": 4030, + "au": 4031, + "Ġnecess": 4032, + "Access": 4033, + "ales": 4034, + "Ġassert": 4035, + "Ġinit": 4036, + "Ph": 4037, + "Ġoptional": 4038, + "Ġassum": 4039, + "ĠUs": 4040, + "Ġfamily": 4041, + "Select": 4042, + "wise": 4043, + "âĶ": 4044, + "Ġpush": 4045, + "