初始化项目，由ModelHub XC社区提供模型
Model: Italianhype/Blum-Finance-4B Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
56
CONTRIBUTING.md
Normal file
56
CONTRIBUTING.md
Normal file
@@ -0,0 +1,56 @@
|
||||
# Contributing Learning Evidence
|
||||
|
||||
BLUM Finance does not collect prompts, outputs, account data, or usage telemetry.
|
||||
Community learning is explicit and evidence-bound.
|
||||
|
||||
## Contribution lifecycle
|
||||
|
||||
1. Run BLUM Finance locally and retain the point-in-time request and response.
|
||||
2. After the stated horizon, attach an observed outcome and verified provenance.
|
||||
3. Create a redacted contribution bundle:
|
||||
|
||||
```bash
|
||||
blum-contribute example.json --output contribution.json --consent
|
||||
```
|
||||
|
||||
4. Inspect the bundle locally. To submit it for review, explicitly run:
|
||||
|
||||
```bash
|
||||
blum-contribute example.json --output contribution.json --consent --push
|
||||
```
|
||||
|
||||
The upload opens a pull request against `Italianhype/Blum-Finance-Memory`.
|
||||
It never writes directly to accepted memory or released model weights.
|
||||
|
||||
## Required evidence
|
||||
|
||||
A contribution must contain:
|
||||
|
||||
- a timestamped request with point-in-time evidence;
|
||||
- the model response generated at that timestamp;
|
||||
- a mature outcome observed after the decision;
|
||||
- verified source provenance and an explicit quality score;
|
||||
- explicit consent under the contribution license.
|
||||
|
||||
Pending, inconclusive, chronologically invalid, tampered, or unverified examples
|
||||
remain quarantined. Secrets, account identifiers, email addresses and Hugging
|
||||
Face tokens are removed from generated bundles.
|
||||
|
||||
## Local memory
|
||||
|
||||
Eligible bundles can improve a local installation without changing weights:
|
||||
|
||||
```bash
|
||||
blum-memory-add contribution.json
|
||||
```
|
||||
|
||||
The inference pipeline retrieves only outcomes observable before the new
|
||||
request's `as_of` timestamp. Retrieved records are labeled as historical
|
||||
analogies and cannot replace current evidence.
|
||||
|
||||
## Model updates
|
||||
|
||||
Accepted records may enter a future immutable dataset snapshot. A training run
|
||||
always creates a challenger. Promotion requires temporal holdout evaluation,
|
||||
no-fabrication and schema checks, adequate sample quality, and an explicit
|
||||
versioned release. Anonymous inputs never self-modify a published model.
|
||||
202
LICENSE
Normal file
202
LICENSE
Normal file
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
164
README.md
Normal file
164
README.md
Normal file
@@ -0,0 +1,164 @@
|
||||
---
|
||||
language:
|
||||
- en
|
||||
- it
|
||||
license: apache-2.0
|
||||
library_name: transformers
|
||||
pipeline_tag: text-generation
|
||||
base_model: Qwen/Qwen3-4B
|
||||
datasets:
|
||||
- Italianhype/Blum-Finance-Reasoning
|
||||
tags:
|
||||
- finance
|
||||
- financial-reasoning
|
||||
- investment-research
|
||||
- risk-management
|
||||
- explainable-ai
|
||||
- qwen3
|
||||
- transformers
|
||||
model-index:
|
||||
- name: BLUM Finance 4B
|
||||
results:
|
||||
- task:
|
||||
type: text-generation
|
||||
name: Multiple-choice financial reasoning
|
||||
dataset:
|
||||
name: MMLU Finance and Business (stratified 100 per subject)
|
||||
type: cais/mmlu
|
||||
config: finance_business_7_subjects
|
||||
split: test
|
||||
metrics:
|
||||
- type: accuracy
|
||||
value: 0.74857143
|
||||
---
|
||||
|
||||
# BLUM Finance 4B — Transformers Benchmark Release
|
||||
|
||||
BLUM Finance 4B is an open, evidence-bound financial reasoning model. This
|
||||
repository is the portable BF16 Transformers release used for public benchmark
|
||||
submission. It contains the same trained LoRA delta as the audited MLX release,
|
||||
mapped and fused into the exact `Qwen/Qwen3-4B` base architecture.
|
||||
|
||||
## Intended Use
|
||||
|
||||
The model structures supplied point-in-time evidence into:
|
||||
|
||||
- balanced bull and bear cases;
|
||||
- explicit risks and invalidation conditions;
|
||||
- conservative confidence;
|
||||
- monitoring conditions and abstention.
|
||||
|
||||
It is not a price oracle, broker, trade executor or proof of market alpha.
|
||||
|
||||
## Load
|
||||
|
||||
```python
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
model_id = "Italianhype/Blum-Finance-4B"
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_id,
|
||||
dtype=torch.bfloat16,
|
||||
device_map="auto",
|
||||
)
|
||||
```
|
||||
|
||||
No `trust_remote_code=True` is required.
|
||||
|
||||
## Governed Continual Memory
|
||||
|
||||
This repository also ships the installable `blum-finance` package. It provides
|
||||
schema-validated inference, auditable local memory and explicit opt-in community
|
||||
contributions:
|
||||
|
||||
```bash
|
||||
pip install "git+https://huggingface.co/Italianhype/Blum-Finance-4B"
|
||||
blum-contribute example.json --output contribution.json --consent
|
||||
blum-memory-add contribution.json
|
||||
```
|
||||
|
||||
Local retrieval accepts only mature, source-verified outcomes observable before
|
||||
the new request timestamp. Memories are labeled historical analogies and cannot
|
||||
replace current evidence. Adding `--push` opens a pull request to the quarantine
|
||||
dataset; inference sends no telemetry and anonymous input never mutates released
|
||||
weights. See `CONTRIBUTING.md` for the evidence contract.
|
||||
|
||||
## Lineage
|
||||
|
||||
- Base: `Qwen/Qwen3-4B`
|
||||
- Base revision: `1cfa9a7208912126459214e8b04321603b3df60c`
|
||||
- Dataset: `Italianhype/Blum-Finance-Reasoning`
|
||||
- Dataset revision: `76ad77699d498fc930daf02e452fe3ec8b490f90`
|
||||
- MLX adapter revision: `ea297ba88ab008e97104b0c118103eef2f8f9ec1`
|
||||
- LoRA rank: `8`
|
||||
- MLX scale: `20`
|
||||
- Equivalent PEFT alpha: `160`
|
||||
- Adapted layers: `20–35`
|
||||
|
||||
The conversion transposes MLX A/B matrices into PEFT orientation and preserves
|
||||
the update `20 × Bᵀ × Aᵀ`. All 112 expected modules were fused; none were
|
||||
missing. The PEFT adapter and merged model produced identical deterministic
|
||||
smoke-test output.
|
||||
|
||||
## Evaluation Status
|
||||
|
||||
The original MLX release passed the 53-example BLUM temporal reasoning test with
|
||||
96.26% aggregate task-contract score, 100% structured validity and 92.86%
|
||||
no-fabrication. These metrics measure BLUM schema adherence and grounding, not
|
||||
general intelligence or trading performance.
|
||||
|
||||
The portable release is published at immutable revision
|
||||
`ad6f5cec7f729370d2976d8c78983521cb37ca83` and tagged
|
||||
`benchmark-submission-v1`.
|
||||
|
||||
The five-example portable conversion smoke test scored 97.32% on the internal
|
||||
task contract, including 100% structured validity and 92.86% no-fabrication.
|
||||
The sample is too small for a robust capability claim.
|
||||
|
||||
Automated submissions were attempted for the Hugging Face Open LLM Leaderboard
|
||||
and the FinOS Open Financial LLM Leaderboard. Both external validators rejected
|
||||
Qwen3 as requiring `trust_remote_code=True`; Transformers 4.57.6 loads this
|
||||
repository without remote code. No official leaderboard score is claimed.
|
||||
|
||||
A deterministic stratified community evaluation on seven finance/business
|
||||
subjects of canonical `cais/mmlu` scored **74.86% accuracy** over 700 test
|
||||
questions (100 per subject), with a Wilson 95% confidence interval of
|
||||
71.51–77.93%. It used five-shot answer-token logit scoring.
|
||||
|
||||
| Subject | Accuracy | Samples |
|
||||
|---|---:|---:|
|
||||
| Business ethics | 77% | 100 |
|
||||
| Econometrics | 65% | 100 |
|
||||
| High-school macroeconomics | 70% | 100 |
|
||||
| High-school microeconomics | 87% | 100 |
|
||||
| Management | 87% | 100 |
|
||||
| Marketing | 92% | 100 |
|
||||
| Professional accounting | 46% | 100 |
|
||||
|
||||
This is an author-run community evaluation, not the full canonical MMLU suite
|
||||
and not an official leaderboard result. The deterministic 100-row cap gives
|
||||
each subject equal weight but can differ from a full-split result. Dataset
|
||||
revision: `c30699e8356da336a370243923dbaf21066bb9fe`.
|
||||
|
||||
[Evaluation summary](evaluations/mmlu-finance-stratified-100/results.json) ·
|
||||
[Per-example predictions](evaluations/mmlu-finance-stratified-100/predictions.jsonl) ·
|
||||
[Evaluator](evaluations/evaluate_mmlu_finance.py)
|
||||
|
||||
Independent evaluation packages are prepared for Vals AI CorpFin/Finance Agent
|
||||
and Scale Labs PRBench Finance. Those scores remain unavailable until the
|
||||
benchmark owners evaluate the immutable model revision.
|
||||
|
||||
## Known Failure
|
||||
|
||||
A sparse-evidence smoke prompt incorrectly identified NVDA as Applied Materials.
|
||||
This is a documented fabrication failure. Callers must validate company identity
|
||||
and all market facts against BLUM Engine or another authoritative data source.
|
||||
|
||||
## Safety
|
||||
|
||||
- Do not use this model as unattended financial advice.
|
||||
- Do not let it authorize or execute transactions.
|
||||
- Supply timestamped evidence and validate every factual statement.
|
||||
- Confidence is bounded but not calibrated on mature trading outcomes.
|
||||
51
adapter/adapter_config.json
Normal file
51
adapter/adapter_config.json
Normal file
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"alpha_pattern": {},
|
||||
"auto_mapping": null,
|
||||
"base_model_name_or_path": "Qwen/Qwen3-4B",
|
||||
"bias": "none",
|
||||
"fan_in_fan_out": false,
|
||||
"inference_mode": true,
|
||||
"init_lora_weights": true,
|
||||
"layer_replication": null,
|
||||
"layers_pattern": "layers",
|
||||
"layers_to_transform": [
|
||||
20,
|
||||
21,
|
||||
22,
|
||||
23,
|
||||
24,
|
||||
25,
|
||||
26,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31,
|
||||
32,
|
||||
33,
|
||||
34,
|
||||
35
|
||||
],
|
||||
"loftq_config": {},
|
||||
"lora_alpha": 160.0,
|
||||
"lora_dropout": 0.0,
|
||||
"megatron_config": null,
|
||||
"megatron_core": "megatron.core",
|
||||
"modules_to_save": null,
|
||||
"peft_type": "LORA",
|
||||
"r": 8,
|
||||
"rank_pattern": {},
|
||||
"revision": null,
|
||||
"target_modules": [
|
||||
"down_proj",
|
||||
"gate_proj",
|
||||
"k_proj",
|
||||
"o_proj",
|
||||
"q_proj",
|
||||
"up_proj",
|
||||
"v_proj"
|
||||
],
|
||||
"task_type": "CAUSAL_LM",
|
||||
"use_dora": false,
|
||||
"use_rslora": false
|
||||
}
|
||||
11
blum_finance/__init__.py
Normal file
11
blum_finance/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
from .inference import BlumFinancePipeline
|
||||
from .memory import BlumFinanceMemoryStore, InvalidMemoryRecord
|
||||
from .schemas import FinancialReasoningRequest, FinancialReasoningResponse
|
||||
|
||||
__all__ = [
|
||||
"BlumFinancePipeline",
|
||||
"BlumFinanceMemoryStore",
|
||||
"InvalidMemoryRecord",
|
||||
"FinancialReasoningRequest",
|
||||
"FinancialReasoningResponse",
|
||||
]
|
||||
237
blum_finance/contributions.py
Normal file
237
blum_finance/contributions.py
Normal file
@@ -0,0 +1,237 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
|
||||
TARGET_REPOSITORY = "Italianhype/Blum-Finance-Memory"
|
||||
BLOCKED_KEYS = {
|
||||
"access_token",
|
||||
"account_id",
|
||||
"api_key",
|
||||
"authorization",
|
||||
"broker_account_id",
|
||||
"refresh_token",
|
||||
}
|
||||
EMAIL_PATTERN = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
|
||||
HF_TOKEN_PATTERN = re.compile(r"\bhf_[A-Za-z0-9_]{8,}\b")
|
||||
|
||||
|
||||
class ConsentRequired(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContributionBundleResult:
|
||||
path: Path
|
||||
content_hash: str
|
||||
uploaded: bool
|
||||
repository: str
|
||||
submission_url: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContributionValidation:
|
||||
accepted: bool
|
||||
blockers: tuple[str, ...]
|
||||
status: str
|
||||
|
||||
|
||||
def build_contribution_bundle(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
output: Path,
|
||||
consent: bool = False,
|
||||
push: bool = False,
|
||||
repository: str = TARGET_REPOSITORY,
|
||||
api: Any | None = None,
|
||||
) -> ContributionBundleResult:
|
||||
if not consent:
|
||||
raise ConsentRequired(
|
||||
"Community contribution is disabled until explicit consent is provided."
|
||||
)
|
||||
sanitized, redactions = _sanitize(payload)
|
||||
canonical = json.dumps(
|
||||
sanitized,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
content_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
bundle = {
|
||||
"schema_version": "blum-finance-contribution-v2",
|
||||
"content_hash": content_hash,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"target_repository": repository,
|
||||
"consent": {
|
||||
"explicit": True,
|
||||
"telemetry_default": "disabled",
|
||||
"license": "cc-by-4.0",
|
||||
},
|
||||
"redactions": redactions,
|
||||
"quarantine_status": "pending_validation",
|
||||
"payload": sanitized,
|
||||
}
|
||||
validation = validate_contribution_bundle(bundle)
|
||||
bundle["quarantine_status"] = validation.status
|
||||
bundle["validation_blockers"] = list(validation.blockers)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(
|
||||
json.dumps(bundle, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
uploaded = False
|
||||
submission_url = None
|
||||
if push:
|
||||
if api is None:
|
||||
from huggingface_hub import HfApi
|
||||
|
||||
api = HfApi()
|
||||
|
||||
result = api.upload_file(
|
||||
path_or_fileobj=str(output),
|
||||
path_in_repo=f"quarantine/{content_hash}.json",
|
||||
repo_id=repository,
|
||||
repo_type="dataset",
|
||||
commit_message=f"contrib: add quarantined example {content_hash[:12]}",
|
||||
create_pr=True,
|
||||
)
|
||||
uploaded = True
|
||||
submission_url = str(
|
||||
getattr(result, "pr_url", None)
|
||||
or getattr(result, "commit_url", None)
|
||||
or result
|
||||
)
|
||||
return ContributionBundleResult(
|
||||
path=output,
|
||||
content_hash=content_hash,
|
||||
uploaded=uploaded,
|
||||
repository=repository,
|
||||
submission_url=submission_url,
|
||||
)
|
||||
|
||||
|
||||
def validate_contribution_bundle(bundle: dict[str, Any]) -> ContributionValidation:
|
||||
blockers: list[str] = []
|
||||
payload = bundle.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
return ContributionValidation(False, ("payload_missing",), "rejected")
|
||||
expected_hash = hashlib.sha256(
|
||||
json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
if not hmac.compare_digest(str(bundle.get("content_hash") or ""), expected_hash):
|
||||
blockers.append("content_hash_mismatch")
|
||||
if (bundle.get("consent") or {}).get("explicit") is not True:
|
||||
blockers.append("explicit_consent_missing")
|
||||
request = payload.get("request")
|
||||
response = payload.get("response")
|
||||
outcome = payload.get("outcome")
|
||||
quality = payload.get("quality")
|
||||
if not isinstance(request, dict) or not request.get("evidence") or not request.get("as_of"):
|
||||
blockers.append("point_in_time_request_missing")
|
||||
if not isinstance(response, dict) or not response.get("thesis"):
|
||||
blockers.append("model_response_missing")
|
||||
if not isinstance(outcome, dict) or not outcome.get("observed_at"):
|
||||
blockers.append("mature_outcome_missing")
|
||||
else:
|
||||
try:
|
||||
decision_at = _parse_datetime((request or {}).get("as_of"))
|
||||
observed_at = _parse_datetime(outcome.get("observed_at"))
|
||||
if observed_at <= decision_at:
|
||||
blockers.append("outcome_chronology_invalid")
|
||||
except (TypeError, ValueError):
|
||||
blockers.append("outcome_timestamp_invalid")
|
||||
if str(outcome.get("status") or "").lower() in {"", "pending", "unresolved", "inconclusive"}:
|
||||
blockers.append("mature_outcome_missing")
|
||||
if not isinstance(quality, dict) or quality.get("source_verified") is not True:
|
||||
blockers.append("source_provenance_unverified")
|
||||
return ContributionValidation(
|
||||
accepted=not blockers,
|
||||
blockers=tuple(dict.fromkeys(blockers)),
|
||||
status="eligible_for_curation" if not blockers else "pending_validation",
|
||||
)
|
||||
|
||||
|
||||
def _sanitize(value: Any) -> tuple[Any, list[str]]:
|
||||
redactions: set[str] = set()
|
||||
|
||||
def clean(item: Any) -> Any:
|
||||
if isinstance(item, dict):
|
||||
result: dict[str, Any] = {}
|
||||
for raw_key, child in item.items():
|
||||
key = str(raw_key)
|
||||
if key.lower() in BLOCKED_KEYS:
|
||||
redactions.add(key.lower())
|
||||
continue
|
||||
result[key] = clean(child)
|
||||
return result
|
||||
if isinstance(item, list):
|
||||
return [clean(child) for child in item]
|
||||
if isinstance(item, str):
|
||||
text = EMAIL_PATTERN.sub(
|
||||
lambda _: _replace(redactions, "email", "[REDACTED_EMAIL]"),
|
||||
item,
|
||||
)
|
||||
return HF_TOKEN_PATTERN.sub(
|
||||
lambda _: _replace(redactions, "hugging_face_token", "[REDACTED_TOKEN]"),
|
||||
text,
|
||||
)
|
||||
return item
|
||||
|
||||
return clean(value), sorted(redactions)
|
||||
|
||||
|
||||
def _replace(redactions: set[str], label: str, replacement: str) -> str:
|
||||
redactions.add(label)
|
||||
return replacement
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> datetime:
|
||||
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Create an explicit, redacted BLUM Finance contribution bundle."
|
||||
)
|
||||
parser.add_argument("input", type=Path)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--consent", action="store_true")
|
||||
parser.add_argument("--push", action="store_true")
|
||||
parser.add_argument("--repository", default=TARGET_REPOSITORY)
|
||||
args = parser.parse_args()
|
||||
payload = json.loads(args.input.read_text(encoding="utf-8"))
|
||||
result = build_contribution_bundle(
|
||||
payload,
|
||||
output=args.output,
|
||||
consent=args.consent,
|
||||
push=args.push,
|
||||
repository=args.repository,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"path": str(result.path),
|
||||
"content_hash": result.content_hash,
|
||||
"uploaded": result.uploaded,
|
||||
"repository": result.repository,
|
||||
"submission_url": result.submission_url,
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
156
blum_finance/inference.py
Normal file
156
blum_finance/inference.py
Normal file
@@ -0,0 +1,156 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Callable, Literal
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from .schemas import FinancialReasoningRequest, FinancialReasoningResponse
|
||||
from .memory import BlumFinanceMemoryStore
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """You are BLUM Finance, an evidence-bound financial reasoning model.
|
||||
Use only the supplied point-in-time evidence. Separate supportive and contradictory
|
||||
evidence. Never invent prices, returns, events or sources. Return one JSON object that
|
||||
matches the requested schema. If evidence is insufficient, abstain explicitly."""
|
||||
|
||||
|
||||
class BlumFinancePipeline:
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "Italianhype/Blum",
|
||||
*,
|
||||
revision: str | None = None,
|
||||
runtime: Literal["transformers", "mlx"] = "transformers",
|
||||
generator: Callable[[list[dict[str, str]]], str] | None = None,
|
||||
memory_store: BlumFinanceMemoryStore | None = None,
|
||||
memory_limit: int = 3,
|
||||
):
|
||||
self.model_id = model_id
|
||||
self.revision = revision
|
||||
self.runtime = runtime
|
||||
self._generator = generator
|
||||
self.memory_store = memory_store
|
||||
self.memory_limit = max(0, int(memory_limit))
|
||||
self._pipeline = None
|
||||
self._mlx_model = None
|
||||
self._mlx_tokenizer = None
|
||||
|
||||
def generate(
|
||||
self,
|
||||
request: FinancialReasoningRequest | dict,
|
||||
) -> FinancialReasoningResponse:
|
||||
parsed_request = (
|
||||
request
|
||||
if isinstance(request, FinancialReasoningRequest)
|
||||
else FinancialReasoningRequest.model_validate(request)
|
||||
)
|
||||
messages = [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
]
|
||||
if self.memory_store is not None and self.memory_limit > 0:
|
||||
memories = self.memory_store.retrieve(parsed_request, limit=self.memory_limit)
|
||||
if memories:
|
||||
messages.append(
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"Validated historical memory follows. It contains past analogies, "
|
||||
"not current market facts. Use it only to challenge the current thesis "
|
||||
"and never copy a past outcome into the present.\n"
|
||||
+ json.dumps(memories, ensure_ascii=False, sort_keys=True)
|
||||
),
|
||||
}
|
||||
)
|
||||
messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": json.dumps(
|
||||
parsed_request.model_dump(mode="json"),
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
raw = self._generator(messages) if self._generator else self._generate(messages)
|
||||
try:
|
||||
payload = _extract_json_object(raw)
|
||||
return FinancialReasoningResponse.model_validate(payload)
|
||||
except (ValueError, json.JSONDecodeError, ValidationError):
|
||||
return FinancialReasoningResponse(
|
||||
status="insufficient_evidence",
|
||||
thesis="The model output could not be validated against the BLUM Finance schema.",
|
||||
confidence=0,
|
||||
what_would_change_the_view=[
|
||||
"Provide a schema-valid response grounded in the supplied evidence."
|
||||
],
|
||||
)
|
||||
|
||||
def _generate(self, messages: list[dict[str, str]]) -> str:
|
||||
if self.runtime == "mlx":
|
||||
return self._generate_with_mlx(messages)
|
||||
return self._generate_with_transformers(messages)
|
||||
|
||||
def _generate_with_transformers(self, messages: list[dict[str, str]]) -> str:
|
||||
if self._pipeline is None:
|
||||
from transformers import pipeline
|
||||
|
||||
self._pipeline = pipeline(
|
||||
"text-generation",
|
||||
model=self.model_id,
|
||||
revision=self.revision,
|
||||
device_map="auto",
|
||||
)
|
||||
result = self._pipeline(
|
||||
messages,
|
||||
max_new_tokens=768,
|
||||
do_sample=False,
|
||||
return_full_text=False,
|
||||
)
|
||||
generated = result[0]["generated_text"]
|
||||
if isinstance(generated, list):
|
||||
generated = generated[-1]["content"]
|
||||
return str(generated)
|
||||
|
||||
def _generate_with_mlx(self, messages: list[dict[str, str]]) -> str:
|
||||
try:
|
||||
from mlx_lm import generate, load
|
||||
from mlx_lm.sample_utils import make_sampler
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"MLX inference requires the 'mlx' optional dependencies on Apple Silicon."
|
||||
) from exc
|
||||
if self._mlx_model is None or self._mlx_tokenizer is None:
|
||||
self._mlx_model, self._mlx_tokenizer = load(
|
||||
self.model_id,
|
||||
revision=self.revision,
|
||||
tokenizer_config={"trust_remote_code": True},
|
||||
)
|
||||
prompt = self._mlx_tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
)
|
||||
return str(
|
||||
generate(
|
||||
self._mlx_model,
|
||||
self._mlx_tokenizer,
|
||||
prompt=prompt,
|
||||
max_tokens=768,
|
||||
sampler=make_sampler(temp=0.0),
|
||||
verbose=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _extract_json_object(text: str) -> dict:
|
||||
stripped = text.strip()
|
||||
if stripped.startswith("```"):
|
||||
stripped = stripped.removeprefix("```json").removeprefix("```")
|
||||
stripped = stripped.removesuffix("```").strip()
|
||||
start = stripped.find("{")
|
||||
end = stripped.rfind("}")
|
||||
if start < 0 or end <= start:
|
||||
raise ValueError("No JSON object found.")
|
||||
return json.loads(stripped[start : end + 1])
|
||||
223
blum_finance/memory.py
Normal file
223
blum_finance/memory.py
Normal file
@@ -0,0 +1,223 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from .schemas import FinancialReasoningRequest, FinancialReasoningResponse
|
||||
|
||||
|
||||
TOKEN_PATTERN = re.compile(r"[A-Za-z0-9_]{2,}")
|
||||
|
||||
|
||||
class InvalidMemoryRecord(ValueError):
|
||||
"""Raised when a record cannot safely become retrieval memory."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StoredMemoryRecord:
|
||||
content_hash: str
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
class BlumFinanceMemoryStore:
|
||||
"""Small, auditable local memory with strict point-in-time retrieval.
|
||||
|
||||
This store never changes model weights. It exposes only matured observations
|
||||
available before a new request and labels them as historical analogies.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path).expanduser().resolve()
|
||||
|
||||
def add(self, payload: dict[str, Any]) -> StoredMemoryRecord:
|
||||
normalized = _validate_memory_payload(payload)
|
||||
content_hash = _content_hash(normalized)
|
||||
row = {"content_hash": content_hash, "payload": normalized}
|
||||
existing = self._rows()
|
||||
if not any(item.get("content_hash") == content_hash for item in existing):
|
||||
self._replace([*existing, row])
|
||||
return StoredMemoryRecord(content_hash=content_hash, payload=normalized)
|
||||
|
||||
def add_bundle(self, bundle: str | Path | dict[str, Any]) -> StoredMemoryRecord:
|
||||
from .contributions import validate_contribution_bundle
|
||||
|
||||
if isinstance(bundle, (str, Path)):
|
||||
value = json.loads(Path(bundle).read_text(encoding="utf-8"))
|
||||
else:
|
||||
value = bundle
|
||||
validation = validate_contribution_bundle(value)
|
||||
if not validation.accepted:
|
||||
raise InvalidMemoryRecord(
|
||||
"Contribution is not eligible for memory: " + ", ".join(validation.blockers)
|
||||
)
|
||||
return self.add(value["payload"])
|
||||
|
||||
def retrieve(
|
||||
self,
|
||||
request: FinancialReasoningRequest,
|
||||
*,
|
||||
limit: int = 3,
|
||||
) -> list[dict[str, Any]]:
|
||||
if limit <= 0:
|
||||
return []
|
||||
request_tokens = _request_tokens(request.model_dump(mode="json"))
|
||||
candidates: list[tuple[float, datetime, dict[str, Any]]] = []
|
||||
for row in self._rows():
|
||||
payload = row.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
try:
|
||||
normalized = _validate_memory_payload(payload)
|
||||
observed_at = _timestamp(normalized["outcome"]["observed_at"])
|
||||
except (InvalidMemoryRecord, KeyError, TypeError, ValueError):
|
||||
continue
|
||||
if observed_at > _aware(request.as_of):
|
||||
continue
|
||||
memory_request = normalized["request"]
|
||||
memory_tokens = _request_tokens(memory_request)
|
||||
overlap = len(request_tokens & memory_tokens) / max(1, len(request_tokens | memory_tokens))
|
||||
ticker_match = str(memory_request.get("ticker", "")).upper() == request.ticker.upper()
|
||||
horizon_match = str(memory_request.get("horizon", "")) == request.horizon
|
||||
score = overlap + (2.0 if ticker_match else 0.0) + (0.5 if horizon_match else 0.0)
|
||||
candidates.append((score, observed_at, _retrieval_payload(normalized, row.get("content_hash"))))
|
||||
candidates.sort(key=lambda item: (item[0], item[1]), reverse=True)
|
||||
return [item[2] for item in candidates[:limit] if item[0] > 0]
|
||||
|
||||
def _rows(self) -> list[dict[str, Any]]:
|
||||
if not self.path.is_file():
|
||||
return []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for line in self.path.read_text(encoding="utf-8").splitlines():
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
rows.append(value)
|
||||
return rows
|
||||
|
||||
def _replace(self, rows: list[dict[str, Any]]) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = self.path.with_suffix(self.path.suffix + ".tmp")
|
||||
body = "".join(_canonical_json(row) + "\n" for row in rows)
|
||||
temporary.write_text(body, encoding="utf-8")
|
||||
os.replace(temporary, self.path)
|
||||
|
||||
|
||||
def _validate_memory_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if not isinstance(payload, dict):
|
||||
raise InvalidMemoryRecord("Memory payload must be an object")
|
||||
request = payload.get("request")
|
||||
response = payload.get("response")
|
||||
outcome = payload.get("outcome")
|
||||
quality = payload.get("quality")
|
||||
try:
|
||||
parsed_request = FinancialReasoningRequest.model_validate(request)
|
||||
parsed_response = FinancialReasoningResponse.model_validate(response)
|
||||
except Exception as exc:
|
||||
raise InvalidMemoryRecord(f"Invalid BLUM request or response: {exc}") from exc
|
||||
if not isinstance(outcome, dict) or not outcome.get("observed_at"):
|
||||
raise InvalidMemoryRecord("A matured outcome with observed_at is required")
|
||||
observed_at = _timestamp(outcome["observed_at"])
|
||||
if observed_at <= _aware(parsed_request.as_of):
|
||||
raise InvalidMemoryRecord("The outcome must be observed after the decision")
|
||||
status = str(outcome.get("status") or "").strip().lower()
|
||||
if status in {"", "pending", "unresolved", "inconclusive"}:
|
||||
raise InvalidMemoryRecord("The outcome is not mature")
|
||||
if not isinstance(quality, dict) or quality.get("source_verified") is not True:
|
||||
raise InvalidMemoryRecord("Memory requires verified source provenance")
|
||||
try:
|
||||
score = float(quality.get("score"))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise InvalidMemoryRecord("Memory quality score is missing") from exc
|
||||
if score < 70:
|
||||
raise InvalidMemoryRecord("Memory quality is below 70")
|
||||
return {
|
||||
**payload,
|
||||
"request": parsed_request.model_dump(mode="json"),
|
||||
"response": parsed_response.model_dump(mode="json"),
|
||||
"outcome": dict(outcome),
|
||||
"quality": {**quality, "score": score},
|
||||
}
|
||||
|
||||
|
||||
def _retrieval_payload(payload: dict[str, Any], content_hash: Any) -> dict[str, Any]:
|
||||
request = payload["request"]
|
||||
response = payload["response"]
|
||||
outcome = payload["outcome"]
|
||||
return {
|
||||
"memory_id": str(content_hash or _content_hash(payload)),
|
||||
"ticker": request.get("ticker"),
|
||||
"horizon": request.get("horizon"),
|
||||
"decision_as_of": request.get("as_of"),
|
||||
"observed_at": outcome.get("observed_at"),
|
||||
"prior_status": response.get("status"),
|
||||
"prior_thesis": response.get("thesis"),
|
||||
"outcome": {
|
||||
key: outcome.get(key)
|
||||
for key in ("status", "realized_r", "benchmark_excess")
|
||||
if outcome.get(key) is not None
|
||||
},
|
||||
"lesson": outcome.get("lesson") or "No explicit lesson was supplied.",
|
||||
"quality_score": payload["quality"]["score"],
|
||||
}
|
||||
|
||||
|
||||
def _request_tokens(payload: dict[str, Any]) -> set[str]:
|
||||
return {
|
||||
token.lower()
|
||||
for token in TOKEN_PATTERN.findall(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||
}
|
||||
|
||||
|
||||
def _content_hash(payload: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _canonical_json(payload: Any) -> str:
|
||||
return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _timestamp(value: Any) -> datetime:
|
||||
if isinstance(value, datetime):
|
||||
return _aware(value)
|
||||
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
return _aware(parsed)
|
||||
|
||||
|
||||
def _aware(value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Import a validated BLUM contribution into local point-in-time memory."
|
||||
)
|
||||
parser.add_argument("bundle", type=Path)
|
||||
parser.add_argument(
|
||||
"--memory",
|
||||
type=Path,
|
||||
default=Path.home() / ".blum-finance" / "memory.jsonl",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
stored = BlumFinanceMemoryStore(args.memory).add_bundle(args.bundle)
|
||||
print(
|
||||
json.dumps(
|
||||
{"status": "stored", "content_hash": stored.content_hash, "memory": str(args.memory)},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
64
blum_finance/schemas.py
Normal file
64
blum_finance/schemas.py
Normal file
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
ReasoningStatus = Literal[
|
||||
"avoid",
|
||||
"watch",
|
||||
"wait_for_trigger",
|
||||
"actionable_if_confirmed",
|
||||
"manage_open_position",
|
||||
"reduce",
|
||||
"exit",
|
||||
"insufficient_evidence",
|
||||
]
|
||||
|
||||
|
||||
class EvidenceItem(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
type: str = Field(min_length=1)
|
||||
value: Any
|
||||
source: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
|
||||
|
||||
class FinancialReasoningRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
ticker: str = Field(min_length=1, max_length=32)
|
||||
as_of: datetime
|
||||
horizon: str = "swing"
|
||||
market_context: dict[str, Any] = Field(default_factory=dict)
|
||||
portfolio_context: dict[str, Any] = Field(default_factory=dict)
|
||||
evidence: list[EvidenceItem] = Field(min_length=1)
|
||||
question: str = "Evaluate the evidence and state what would change the view."
|
||||
|
||||
|
||||
class FinancialReasoningResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
status: ReasoningStatus
|
||||
thesis: str = Field(min_length=1)
|
||||
bull_case: list[str] = Field(default_factory=list)
|
||||
bear_case: list[str] = Field(default_factory=list)
|
||||
risks: list[str] = Field(default_factory=list)
|
||||
invalidation_conditions: list[str] = Field(default_factory=list)
|
||||
confidence: float = Field(ge=0, le=100)
|
||||
what_would_change_the_view: list[str] = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_risk_definition_for_actionable_states(self) -> "FinancialReasoningResponse":
|
||||
actionable = {
|
||||
"actionable_if_confirmed",
|
||||
"manage_open_position",
|
||||
"reduce",
|
||||
"exit",
|
||||
}
|
||||
if self.status in actionable and (not self.risks or not self.invalidation_conditions):
|
||||
raise ValueError("Actionable states require risks and invalidation conditions.")
|
||||
return self
|
||||
31
config.json
Normal file
31
config.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"_name_or_path": "Italianhype/Blum-Finance-4B",
|
||||
"architectures": [
|
||||
"Qwen3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151645,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 2560,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 9728,
|
||||
"max_position_embeddings": 40960,
|
||||
"max_window_layers": 36,
|
||||
"model_type": "qwen3",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 36,
|
||||
"num_key_value_heads": 8,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 1000000,
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.51.0",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
30
conversion/manifest.json
Normal file
30
conversion/manifest.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"artifact_hashes": {
|
||||
"LICENSE": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
|
||||
"README.md": "f126b7d8e010573c584b4a3ba9d10ed8de7b405215aae23ac8615c61b7617277",
|
||||
"adapter/adapter_config.json": "654d90b8d4b6c419e574e68861baf9351b2cf19a57e0aa800aecab02a5405c56",
|
||||
"adapter/adapter_model.safetensors": "d3a0771e5efbf046ad0c7e1070e256f8b819bc0418b1c550481391f7e6295fb2",
|
||||
"config.json": "b5c438db998a4c4b58ce30ed7ea666394bf9b823edc3d569cd9721816fc37de8",
|
||||
"generation_config.json": "2325da0f15bb848e018c5ae071b7943332e9f871d6b60e2ed22ca97d4cb993d2",
|
||||
"merges.txt": "8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5",
|
||||
"model-00001-of-00003.safetensors": "328a91d3122359d5547f9d79521205bc0a46e1f79a792dfe650e99fc2d651223",
|
||||
"model-00002-of-00003.safetensors": "35aec6479cf06b36aead31759beaf208bd2db943aac03a2178fa3c437229201f",
|
||||
"model-00003-of-00003.safetensors": "3c49e27698488893b03e263040597a617c2caca7047ef6925343e7eb79d3b76e",
|
||||
"model.safetensors.index.json": "6dc0981b8829fead746441f68f38f24c5ca4a3a66351f652c26c6df0efc43ab2",
|
||||
"tokenizer.json": "aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4",
|
||||
"tokenizer_config.json": "d5d09f07b48c3086c508b30d1c9114bd1189145b74e982a265350c923acd8101",
|
||||
"vocab.json": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"
|
||||
},
|
||||
"base_model": "Qwen/Qwen3-4B",
|
||||
"base_revision": "1cfa9a7208912126459214e8b04321603b3df60c",
|
||||
"dataset_revision": "76ad77699d498fc930daf02e452fe3ec8b490f90",
|
||||
"deterministic_peft_merged_output_match": true,
|
||||
"known_sparse_evidence_identity_failure": true,
|
||||
"merged_modules": 112,
|
||||
"missing_modules": [],
|
||||
"mlx_adapter_revision": "ea297ba88ab008e97104b0c118103eef2f8f9ec1",
|
||||
"mlx_scale": 20.0,
|
||||
"peft_alpha": 160.0,
|
||||
"rank": 8,
|
||||
"schema_version": "blum-transformers-conversion-v1"
|
||||
}
|
||||
96
evaluations/EXTERNAL_BENCHMARK_SUBMISSIONS.md
Normal file
96
evaluations/EXTERNAL_BENCHMARK_SUBMISSIONS.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# BLUM Finance 4B External Benchmark Submission
|
||||
|
||||
## Immutable Candidate
|
||||
|
||||
- Model: `Italianhype/Blum-Finance-4B`
|
||||
- Revision: `ad6f5cec7f729370d2976d8c78983521cb37ca83`
|
||||
- Tag: `benchmark-submission-v1`
|
||||
- Base model: `Qwen/Qwen3-4B`
|
||||
- License: Apache-2.0
|
||||
- Parameters: 4.0B
|
||||
- Weights: merged BF16 Safetensors
|
||||
- Library: Hugging Face Transformers
|
||||
- Remote model code: not required with Transformers 4.57.6
|
||||
- Intended task: evidence-bound financial reasoning
|
||||
|
||||
## Requested Independent Evaluations
|
||||
|
||||
### Vals AI
|
||||
|
||||
Requested suites:
|
||||
|
||||
1. CorpFin v2
|
||||
2. Finance Agent v2
|
||||
3. Vals Index finance components, if eligible
|
||||
|
||||
Vals AI runs proprietary evaluations independently. A public leaderboard score
|
||||
cannot be self-published. New or custom models require contact with the Vals
|
||||
team through `contact@vals.ai` or the Vals platform. The model should be
|
||||
identified by the immutable Hub revision above.
|
||||
|
||||
### Scale Labs
|
||||
|
||||
Requested suite:
|
||||
|
||||
1. Professional Reasoning Benchmark - Finance
|
||||
|
||||
Scale Labs asks model providers to contact `leaderboards@scale.com`. To preserve
|
||||
leaderboard integrity, the first featured run must occur before the organization
|
||||
encounters the private prompts. BLUM has not downloaded or used hidden PRBench
|
||||
evaluation prompts.
|
||||
|
||||
## Reproducible Inference Configuration
|
||||
|
||||
```python
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
model_id = "Italianhype/Blum-Finance-4B"
|
||||
revision = "ad6f5cec7f729370d2976d8c78983521cb37ca83"
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_id,
|
||||
revision=revision,
|
||||
dtype=torch.bfloat16,
|
||||
device_map="auto",
|
||||
)
|
||||
```
|
||||
|
||||
For conversational evaluation, use the repository chat template. Disable
|
||||
Qwen's reasoning envelope only when a benchmark requires answer-only output:
|
||||
|
||||
```python
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
)
|
||||
```
|
||||
|
||||
Use each benchmark owner's required temperature, token budget and tool policy.
|
||||
Do not substitute BLUM's internal evaluation settings for the official
|
||||
methodology.
|
||||
|
||||
## Submission Message
|
||||
|
||||
Subject: `Open-weight finance model evaluation request — BLUM Finance 4B`
|
||||
|
||||
> BLUM Finance 4B is an Apache-2.0, Qwen3-based 4B open-weight model specialized
|
||||
> in evidence-bound financial reasoning, contradiction handling, risk
|
||||
> disclosure and explicit invalidation. We request independent evaluation of
|
||||
> immutable revision
|
||||
> `ad6f5cec7f729370d2976d8c78983521cb37ca83` from
|
||||
> `Italianhype/Blum-Finance-4B`. The repository uses standard Transformers and
|
||||
> merged BF16 Safetensors without custom model code. We will publish favorable
|
||||
> or unfavorable results without altering them and will not claim trading alpha
|
||||
> from language-model benchmark performance.
|
||||
|
||||
## Integrity Rules
|
||||
|
||||
- Never call a self-run result an official Vals or Scale score.
|
||||
- Never tune on private or held-out leaderboard prompts.
|
||||
- Keep the submitted revision immutable.
|
||||
- Publish failures and confidence intervals.
|
||||
- Keep language-model capability separate from paper-forward trading evidence.
|
||||
414
evaluations/evaluate_mmlu_finance.py
Normal file
414
evaluations/evaluate_mmlu_finance.py
Normal file
@@ -0,0 +1,414 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate a causal language model on the finance-related MMLU subjects."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
import pyarrow.parquet as parquet
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
|
||||
DATASET_ID = "cais/mmlu"
|
||||
DATASET_REVISION = "c30699e8356da336a370243923dbaf21066bb9fe"
|
||||
DEFAULT_SUBJECTS = (
|
||||
"business_ethics",
|
||||
"econometrics",
|
||||
"high_school_macroeconomics",
|
||||
"high_school_microeconomics",
|
||||
"management",
|
||||
"marketing",
|
||||
"professional_accounting",
|
||||
)
|
||||
ANSWER_LABELS = ("A", "B", "C", "D")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MmluExample:
|
||||
question: str
|
||||
choices: tuple[str, ...]
|
||||
answer: int
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
parser.add_argument("--cache-dir", default=".artifacts/mmlu-finance")
|
||||
parser.add_argument("--revision", default=None)
|
||||
parser.add_argument("--subjects", nargs="+", default=list(DEFAULT_SUBJECTS))
|
||||
parser.add_argument("--few-shot", type=int, default=5)
|
||||
parser.add_argument("--batch-size", type=int, default=2)
|
||||
parser.add_argument("--max-samples-per-subject", type=int, default=None)
|
||||
parser.add_argument("--device", choices=("auto", "cpu", "cuda", "mps"), default="auto")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def resolve_device(requested: str) -> str:
|
||||
if requested != "auto":
|
||||
return requested
|
||||
if torch.cuda.is_available():
|
||||
return "cuda"
|
||||
if torch.backends.mps.is_available():
|
||||
return "mps"
|
||||
return "cpu"
|
||||
|
||||
|
||||
def dataset_url(subject: str, split: str) -> str:
|
||||
filename = f"{split}-00000-of-00001.parquet"
|
||||
return (
|
||||
f"https://huggingface.co/datasets/{DATASET_ID}/resolve/"
|
||||
f"{DATASET_REVISION}/{subject}/{filename}"
|
||||
)
|
||||
|
||||
|
||||
def download_file(url: str, destination: Path, attempts: int = 6) -> None:
|
||||
if destination.exists() and destination.stat().st_size > 0:
|
||||
return
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = destination.with_suffix(destination.suffix + ".part")
|
||||
last_error: BaseException | None = None
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
with (
|
||||
urllib.request.urlopen(url, timeout=120) as response,
|
||||
temporary.open("wb") as output,
|
||||
):
|
||||
while chunk := response.read(1024 * 1024):
|
||||
output.write(chunk)
|
||||
temporary.replace(destination)
|
||||
return
|
||||
except (OSError, urllib.error.URLError) as exc:
|
||||
last_error = exc
|
||||
temporary.unlink(missing_ok=True)
|
||||
if attempt == attempts:
|
||||
break
|
||||
delay = min(2 ** (attempt - 1), 30)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"download_retry": attempt,
|
||||
"delay_seconds": delay,
|
||||
"url": url,
|
||||
"error": str(exc),
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(delay)
|
||||
raise RuntimeError(f"Unable to download {url} after {attempts} attempts") from last_error
|
||||
|
||||
|
||||
def load_split(cache_dir: Path, subject: str, split: str) -> list[MmluExample]:
|
||||
path = cache_dir / subject / f"{split}.parquet"
|
||||
download_file(dataset_url(subject, split), path)
|
||||
records = parquet.read_table(path).to_pylist()
|
||||
return [
|
||||
MmluExample(
|
||||
question=str(record["question"]),
|
||||
choices=tuple(str(choice) for choice in record["choices"]),
|
||||
answer=int(record["answer"]),
|
||||
)
|
||||
for record in records
|
||||
]
|
||||
|
||||
|
||||
def format_example(example: MmluExample, include_answer: bool) -> str:
|
||||
lines = [example.question]
|
||||
lines.extend(
|
||||
f"{label}. {choice}"
|
||||
for label, choice in zip(ANSWER_LABELS, example.choices, strict=True)
|
||||
)
|
||||
if include_answer:
|
||||
lines.append(f"Answer: {ANSWER_LABELS[example.answer]}")
|
||||
else:
|
||||
lines.append("Answer:")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_prompt(subject: str, few_shot: Iterable[MmluExample], test: MmluExample) -> str:
|
||||
readable_subject = subject.replace("_", " ")
|
||||
header = (
|
||||
"The following are multiple choice questions (with answers) "
|
||||
f"about {readable_subject}.\n\n"
|
||||
)
|
||||
demonstrations = "\n\n".join(
|
||||
format_example(example, include_answer=True) for example in few_shot
|
||||
)
|
||||
return f"{header}{demonstrations}\n\n{format_example(test, include_answer=False)}"
|
||||
|
||||
|
||||
def answer_token_ids(tokenizer: Any) -> list[int]:
|
||||
result: list[int] = []
|
||||
for label in ANSWER_LABELS:
|
||||
encoded = tokenizer.encode(f" {label}", add_special_tokens=False)
|
||||
if len(encoded) != 1:
|
||||
raise ValueError(f"Answer label {label!r} is not a single token: {encoded}")
|
||||
result.append(encoded[0])
|
||||
return result
|
||||
|
||||
|
||||
def chunks(values: list[Any], size: int) -> Iterable[list[Any]]:
|
||||
for start in range(0, len(values), size):
|
||||
yield values[start : start + size]
|
||||
|
||||
|
||||
def write_json_atomic(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".part")
|
||||
temporary.write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def write_jsonl_atomic(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".part")
|
||||
with temporary.open("w", encoding="utf-8") as output:
|
||||
for row in rows:
|
||||
output.write(json.dumps(row, sort_keys=True) + "\n")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def load_subject_checkpoint(
|
||||
output_dir: Path,
|
||||
subject: str,
|
||||
expected_config: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], list[dict[str, Any]]] | None:
|
||||
subject_dir = output_dir / "subjects" / subject
|
||||
result_path = subject_dir / "result.json"
|
||||
predictions_path = subject_dir / "predictions.jsonl"
|
||||
if not result_path.exists() or not predictions_path.exists():
|
||||
return None
|
||||
result = json.loads(result_path.read_text(encoding="utf-8"))
|
||||
predictions = [
|
||||
json.loads(line)
|
||||
for line in predictions_path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
if result.get("sample_size") != len(predictions):
|
||||
return None
|
||||
if result.get("evaluation_config") != expected_config:
|
||||
return None
|
||||
return result, predictions
|
||||
|
||||
|
||||
def write_subject_checkpoint(
|
||||
output_dir: Path,
|
||||
subject: str,
|
||||
result: dict[str, Any],
|
||||
predictions: list[dict[str, Any]],
|
||||
) -> None:
|
||||
subject_dir = output_dir / "subjects" / subject
|
||||
write_json_atomic(subject_dir / "result.json", result)
|
||||
write_jsonl_atomic(subject_dir / "predictions.jsonl", predictions)
|
||||
|
||||
|
||||
def wilson_interval(correct: int, total: int, z: float = 1.959963984540054) -> tuple[float, float]:
|
||||
if total == 0:
|
||||
return 0.0, 0.0
|
||||
proportion = correct / total
|
||||
denominator = 1 + (z * z / total)
|
||||
centre = proportion + z * z / (2 * total)
|
||||
margin = z * math.sqrt((proportion * (1 - proportion) + z * z / (4 * total)) / total)
|
||||
return (centre - margin) / denominator, (centre + margin) / denominator
|
||||
|
||||
|
||||
def evaluate_subject(
|
||||
*,
|
||||
model: Any,
|
||||
tokenizer: Any,
|
||||
device: str,
|
||||
subject: str,
|
||||
dev: list[MmluExample],
|
||||
test: list[MmluExample],
|
||||
few_shot_count: int,
|
||||
batch_size: int,
|
||||
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
few_shot = dev[:few_shot_count]
|
||||
answer_ids = torch.tensor(answer_token_ids(tokenizer), device=device)
|
||||
predictions: list[dict[str, Any]] = []
|
||||
started = time.perf_counter()
|
||||
|
||||
for batch in chunks(test, batch_size):
|
||||
prompts = [build_prompt(subject, few_shot, example) for example in batch]
|
||||
encoded = tokenizer(
|
||||
prompts,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
add_special_tokens=True,
|
||||
).to(device)
|
||||
with torch.inference_mode():
|
||||
logits = model(**encoded).logits
|
||||
# Left padding keeps every final prompt token at the final sequence index.
|
||||
final_positions = torch.full(
|
||||
(len(batch),),
|
||||
encoded["input_ids"].shape[1] - 1,
|
||||
device=device,
|
||||
dtype=torch.long,
|
||||
)
|
||||
row_indices = torch.arange(len(batch), device=device)
|
||||
final_logits = logits[row_indices, final_positions]
|
||||
choice_logits = final_logits.index_select(dim=1, index=answer_ids)
|
||||
probabilities = torch.softmax(choice_logits.float(), dim=1).cpu()
|
||||
predicted = choice_logits.argmax(dim=1).cpu().tolist()
|
||||
|
||||
for example, prediction, probability in zip(
|
||||
batch, predicted, probabilities.tolist(), strict=True
|
||||
):
|
||||
predictions.append(
|
||||
{
|
||||
"subject": subject,
|
||||
"expected": ANSWER_LABELS[example.answer],
|
||||
"predicted": ANSWER_LABELS[prediction],
|
||||
"correct": prediction == example.answer,
|
||||
"choice_probabilities": {
|
||||
label: round(value, 8)
|
||||
for label, value in zip(ANSWER_LABELS, probability, strict=True)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
correct = sum(int(row["correct"]) for row in predictions)
|
||||
total = len(predictions)
|
||||
lower, upper = wilson_interval(correct, total)
|
||||
return (
|
||||
{
|
||||
"subject": subject,
|
||||
"correct": correct,
|
||||
"sample_size": total,
|
||||
"accuracy": round(correct / total, 8) if total else None,
|
||||
"confidence_interval_95": [round(lower, 8), round(upper, 8)],
|
||||
"duration_seconds": round(time.perf_counter() - started, 3),
|
||||
},
|
||||
predictions,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.few_shot < 0 or args.batch_size < 1:
|
||||
raise ValueError("few-shot must be non-negative and batch-size must be positive")
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
cache_dir = Path(args.cache_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
device = resolve_device(args.device)
|
||||
dtype = torch.bfloat16 if device in {"cuda", "mps"} else torch.float32
|
||||
|
||||
# Resolve every dataset dependency before allocating model memory. A transient
|
||||
# CDN failure must not invalidate hours of completed inference.
|
||||
splits: dict[str, tuple[list[MmluExample], list[MmluExample]]] = {}
|
||||
for subject in args.subjects:
|
||||
dev = load_split(cache_dir, subject, "dev")
|
||||
test = load_split(cache_dir, subject, "test")
|
||||
if args.max_samples_per_subject is not None:
|
||||
test = test[: args.max_samples_per_subject]
|
||||
splits[subject] = (dev, test)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision)
|
||||
if tokenizer.pad_token_id is None:
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
tokenizer.padding_side = "left"
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
args.model,
|
||||
revision=args.revision,
|
||||
dtype=dtype,
|
||||
low_cpu_mem_usage=True,
|
||||
).to(device)
|
||||
model.eval()
|
||||
|
||||
subject_results: list[dict[str, Any]] = []
|
||||
predictions: list[dict[str, Any]] = []
|
||||
for subject in args.subjects:
|
||||
dev, test = splits[subject]
|
||||
evaluation_config = {
|
||||
"dataset_revision": DATASET_REVISION,
|
||||
"few_shot": args.few_shot,
|
||||
"model_revision": args.revision,
|
||||
"sample_size": len(test),
|
||||
}
|
||||
checkpoint = load_subject_checkpoint(
|
||||
output_dir,
|
||||
subject,
|
||||
evaluation_config,
|
||||
)
|
||||
if checkpoint is None:
|
||||
result, subject_predictions = evaluate_subject(
|
||||
model=model,
|
||||
tokenizer=tokenizer,
|
||||
device=device,
|
||||
subject=subject,
|
||||
dev=dev,
|
||||
test=test,
|
||||
few_shot_count=args.few_shot,
|
||||
batch_size=args.batch_size,
|
||||
)
|
||||
result["evaluation_config"] = evaluation_config
|
||||
write_subject_checkpoint(
|
||||
output_dir,
|
||||
subject,
|
||||
result,
|
||||
subject_predictions,
|
||||
)
|
||||
else:
|
||||
result, subject_predictions = checkpoint
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"subject": subject,
|
||||
"status": "resumed_from_checkpoint",
|
||||
"sample_size": result["sample_size"],
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
subject_results.append(result)
|
||||
predictions.extend(subject_predictions)
|
||||
print(json.dumps(result, sort_keys=True), flush=True)
|
||||
|
||||
total = sum(row["sample_size"] for row in subject_results)
|
||||
correct = sum(row["correct"] for row in subject_results)
|
||||
lower, upper = wilson_interval(correct, total)
|
||||
accuracies = [
|
||||
row["accuracy"] for row in subject_results if row["accuracy"] is not None
|
||||
]
|
||||
summary = {
|
||||
"benchmark": "MMLU finance and business subset",
|
||||
"dataset": DATASET_ID,
|
||||
"dataset_revision": DATASET_REVISION,
|
||||
"model": args.model,
|
||||
"model_revision": args.revision,
|
||||
"evaluated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"device": device,
|
||||
"dtype": str(dtype).replace("torch.", ""),
|
||||
"few_shot": args.few_shot,
|
||||
"sample_limit_per_subject": args.max_samples_per_subject,
|
||||
"subjects": subject_results,
|
||||
"sample_size": total,
|
||||
"micro_accuracy": round(correct / total, 8) if total else None,
|
||||
"macro_accuracy": round(statistics.mean(accuracies), 8) if accuracies else None,
|
||||
"confidence_interval_95": [round(lower, 8), round(upper, 8)],
|
||||
"status": "community_evaluation",
|
||||
"official_leaderboard_result": False,
|
||||
}
|
||||
write_json_atomic(output_dir / "results.json", summary)
|
||||
write_jsonl_atomic(output_dir / "predictions.jsonl", predictions)
|
||||
print(json.dumps(summary, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
700
evaluations/mmlu-finance-stratified-100/predictions.jsonl
Normal file
700
evaluations/mmlu-finance-stratified-100/predictions.jsonl
Normal file
@@ -0,0 +1,700 @@
|
||||
{"choice_probabilities": {"A": 0.98187572, "B": 5.724e-05, "C": 0.01798368, "D": 8.328e-05}, "correct": false, "expected": "C", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00043023, "B": 0.99880457, "C": 0.00033506, "D": 0.00043023}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00037758, "B": 0.00042785, "C": 0.00590631, "D": 0.99328834}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00094425, "B": 0.00023874, "C": 0.08499837, "D": 0.9138186}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 7.468e-05, "B": 0.99766469, "C": 0.00033468, "D": 0.00192595}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.02550781, "B": 0.84470278, "C": 0.01547127, "D": 0.1143181}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.01581277, "B": 0.86334777, "C": 0.11684143, "D": 0.00399809}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.9996717, "B": 0.00010887, "C": 0.00012337, "D": 9.608e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99613935, "B": 0.00102931, "C": 0.001923, "D": 0.00090836}, "correct": false, "expected": "B", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.08075828, "B": 0.76621324, "C": 0.15087633, "D": 0.00215214}, "correct": false, "expected": "C", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.01396109, "B": 0.00581985, "C": 0.97874761, "D": 0.00147149}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00974503, "B": 0.06354561, "C": 0.87721997, "D": 0.04948936}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00705921, "B": 0.06697597, "C": 0.00139004, "D": 0.92457467}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00358426, "B": 0.99381644, "C": 0.00090624, "D": 0.00169309}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.17858176, "B": 0.01882238, "C": 0.00224801, "D": 0.80034792}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00116814, "B": 0.99766147, "C": 0.00026065, "D": 0.00090975}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99609178, "B": 0.00062428, "C": 0.00048619, "D": 0.00279782}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00043036, "B": 0.00023035, "C": 0.99910897, "D": 0.00023035}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00020271, "B": 0.00317098, "C": 0.99629211, "D": 0.00033422}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.73684645, "B": 0.12804471, "C": 0.04710502, "D": 0.08800376}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.9860518, "B": 0.00022734, "C": 0.01241256, "D": 0.00130827}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 5.827e-05, "B": 0.9995876, "C": 0.00029592, "D": 5.827e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00132445, "B": 0.00033487, "C": 0.99824476, "D": 9.594e-05}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99908113, "B": 0.00048765, "C": 0.00033515, "D": 9.602e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99749458, "B": 0.00037918, "C": 0.00080272, "D": 0.00132346}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 9.609e-05, "B": 0.99972302, "C": 9.609e-05, "D": 8.479e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00023018, "B": 0.99834073, "C": 0.0008034, "D": 0.00062569}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00026116, "B": 6.603e-05, "C": 0.99963737, "D": 3.534e-05}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00026773, "B": 0.62154579, "C": 0.00119987, "D": 0.37698659}, "correct": false, "expected": "D", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 1.67e-05, "B": 3.12e-05, "C": 0.00013982, "D": 0.9998123}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99791938, "B": 0.00026071, "C": 0.00090998, "D": 0.00090998}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.98714346, "B": 0.0124263, "C": 0.00029224, "D": 0.00013804}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.66848576, "B": 0.14915934, "C": 0.10251562, "D": 0.07983924}, "correct": false, "expected": "D", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99048084, "B": 0.00756241, "C": 0.00115973, "D": 0.00079707}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.06687846, "B": 0.00905102, "C": 0.00084188, "D": 0.92322868}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00054931, "B": 0.00590559, "C": 0.99316758, "D": 0.00037753}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.93060362, "B": 0.0674127, "C": 0.00074889, "D": 0.00123471}, "correct": false, "expected": "B", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00029582, "B": 0.00026106, "C": 0.00020331, "D": 0.9992398}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00149947, "B": 0.00090948, "C": 0.99736112, "D": 0.00022995}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 1.013e-05, "B": 0.99975663, "C": 0.00015842, "D": 7.483e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.61844701, "B": 0.12177943, "C": 0.12177943, "D": 0.13799419}, "correct": false, "expected": "D", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.72196198, "B": 0.09770694, "C": 0.01923963, "D": 0.16109149}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.98437542, "B": 0.0075158, "C": 0.00355021, "D": 0.00455856}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 2.131e-05, "B": 0.99345618, "C": 0.00406003, "D": 0.00246253}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00892534, "B": 0.06594984, "C": 0.91040945, "D": 0.0147154}, "correct": false, "expected": "B", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00020322, "B": 0.00012326, "C": 0.00091075, "D": 0.99876285}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99244988, "B": 0.00131676, "C": 0.0052079, "D": 0.0010255}, "correct": false, "expected": "C", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99984717, "B": 3.535e-05, "C": 6.605e-05, "D": 5.144e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.0002611, "B": 9.605e-05, "C": 0.0002611, "D": 0.99938178}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00279917, "B": 0.99657506, "C": 0.00048642, "D": 0.00013936}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99894243, "B": 0.00043029, "C": 0.0005525, "D": 7.477e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.0096931, "B": 0.9887234, "C": 0.00115768, "D": 0.00042589}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00191977, "B": 0.00169419, "C": 0.11877213, "D": 0.87761384}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00970238, "B": 0.00020137, "C": 0.98966998, "D": 0.00042629}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.92390656, "B": 0.00622523, "C": 0.06692757, "D": 0.00294059}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.65395439, "B": 0.12877126, "C": 0.12877126, "D": 0.08850311}, "correct": false, "expected": "C", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00043044, "B": 7.48e-05, "C": 0.99929142, "D": 0.00020332}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99975103, "B": 2.429e-05, "C": 8.48e-05, "D": 0.00013981}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.02458242, "B": 0.04592601, "C": 0.92244858, "D": 0.00704298}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00145452, "B": 0.00047221, "C": 0.75346041, "D": 0.24461278}, "correct": false, "expected": "D", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 5.141e-05, "B": 0.00043042, "C": 0.00026106, "D": 0.99925715}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.9942469, "B": 0.0002023, "C": 0.00033353, "D": 0.00521733}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.66644377, "B": 0.21636261, "C": 0.03759819, "D": 0.07959536}, "correct": false, "expected": "B", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99285179, "B": 0.00590371, "C": 0.00062225, "D": 0.00062225}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00339809, "B": 0.94219667, "C": 0.00125009, "D": 0.0531551}, "correct": false, "expected": "A", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.64746791, "B": 0.00160491, "C": 0.34656462, "D": 0.0043626}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00668672, "B": 0.00029379, "C": 0.00062196, "D": 0.99239761}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99926299, "B": 0.00037985, "C": 0.00026107, "D": 9.604e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99803585, "B": 0.00170027, "C": 0.00017921, "D": 8.465e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.61728239, "B": 0.09466332, "C": 0.25732157, "D": 0.03073268}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.4888567, "B": 0.08495056, "C": 0.38072199, "D": 0.04547076}, "correct": false, "expected": "B", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.70112848, "B": 0.15644291, "C": 0.03490713, "D": 0.10752154}, "correct": false, "expected": "B", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.81432122, "B": 0.00138729, "C": 0.18169963, "D": 0.00259181}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.9910149, "B": 0.0007975, "C": 0.00756649, "D": 0.0006211}, "correct": false, "expected": "C", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.01791037, "B": 0.97787273, "C": 0.00274664, "D": 0.00147017}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.65009987, "B": 0.00067175, "C": 0.34797341, "D": 0.00125499}, "correct": false, "expected": "C", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99591511, "B": 0.00029484, "C": 0.00132136, "D": 0.00246863}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00029556, "B": 0.00070902, "C": 0.99836975, "D": 0.0006257}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.96175766, "B": 0.02904256, "C": 0.00734311, "D": 0.00185663}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.84047192, "B": 0.12889054, "C": 0.0082397, "D": 0.02239782}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.29902339, "B": 0.43507645, "C": 0.14124864, "D": 0.12465149}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00187502, "B": 0.13144884, "C": 0.85715401, "D": 0.00952212}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00148969, "B": 0.00245608, "C": 0.00519953, "D": 0.99085474}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.01781676, "B": 0.00100515, "C": 0.00841604, "D": 0.97276211}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.74669874, "B": 0.12975678, "C": 0.05409068, "D": 0.0694538}, "correct": false, "expected": "C", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.4627524, "B": 0.21858877, "C": 0.24769352, "D": 0.07096538}, "correct": false, "expected": "C", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99516171, "B": 0.00116521, "C": 0.00149617, "D": 0.00217691}, "correct": false, "expected": "C", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99911124, "B": 0.00033516, "C": 0.0001233, "D": 0.00043036}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 2.429e-05, "B": 0.99951124, "C": 0.00026113, "D": 0.00020337}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00116814, "B": 0.99766147, "C": 0.00090975, "D": 0.00026065}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.44241941, "B": 0.30407012, "C": 0.12675528, "D": 0.12675528}, "correct": false, "expected": "B", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.4805122, "B": 0.09461846, "C": 0.09461846, "D": 0.33025089}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.64273292, "B": 0.12656161, "C": 0.18414608, "D": 0.04655942}, "correct": false, "expected": "D", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.01792974, "B": 0.00166773, "C": 0.00147176, "D": 0.97893083}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.46183047, "B": 0.31741112, "C": 0.1499344, "D": 0.070824}, "correct": false, "expected": "B", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.6498673, "B": 0.00142158, "C": 0.34784892, "D": 0.00086223}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 7.331e-05, "B": 0.02032771, "C": 0.97944379, "D": 0.0001552}, "correct": true, "expected": "C", "predicted": "C", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.39560065, "B": 0.23994392, "C": 0.0153391, "D": 0.34911636}, "correct": true, "expected": "A", "predicted": "A", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.00029348, "B": 0.00079776, "C": 0.00756897, "D": 0.99133974}, "correct": true, "expected": "D", "predicted": "D", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 8.94e-06, "B": 0.99984956, "C": 9.61e-05, "D": 4.539e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "business_ethics"}
|
||||
{"choice_probabilities": {"A": 0.99884677, "B": 0.00070936, "C": 0.00033508, "D": 0.00010878}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00102832, "B": 0.00316744, "C": 0.99518061, "D": 0.00062371}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.99547195, "B": 0.00217759, "C": 0.00192171, "D": 0.00042879}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00042797, "B": 0.00079955, "C": 0.00521372, "D": 0.9935587}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.000261, "B": 0.00023033, "C": 0.99902093, "D": 0.00048762}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.1803731, "B": 0.80837613, "C": 0.00898025, "D": 0.00227056}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 2.753e-05, "B": 0.99977869, "C": 0.00015843, "D": 3.535e-05}, "correct": false, "expected": "A", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00191447, "B": 0.00116119, "C": 0.00520407, "D": 0.99172032}, "correct": false, "expected": "C", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00192187, "B": 0.9955526, "C": 0.00149675, "D": 0.0010287}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01922535, "B": 0.34077755, "C": 0.63665622, "D": 0.00334087}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00315659, "B": 0.99177223, "C": 0.00315659, "D": 0.00191457}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01376609, "B": 0.00348061, "C": 0.01767601, "D": 0.96507728}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00508851, "B": 0.02012543, "C": 0.00508851, "D": 0.96969754}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.98084027, "B": 0.01089615, "C": 0.00243126, "D": 0.00583229}, "correct": false, "expected": "B", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01233751, "B": 0.98009014, "C": 0.00514304, "D": 0.0024294}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.98238832, "B": 0.01091335, "C": 0.00454936, "D": 0.00214897}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.8913126, "B": 0.09394366, "C": 0.00600562, "D": 0.00873812}, "correct": false, "expected": "C", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00020332, "B": 0.9992575, "C": 0.00048773, "D": 5.141e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00544479, "B": 0.80807865, "C": 0.00616976, "D": 0.18030672}, "correct": false, "expected": "D", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.19739848, "B": 0.25346467, "C": 0.22368179, "D": 0.32545507}, "correct": false, "expected": "B", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 6.602e-05, "B": 0.99938786, "C": 0.0004878, "D": 5.826e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.31402552, "B": 0.01379732, "C": 0.00738517, "D": 0.664792}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.02008031, "B": 0.00164829, "C": 0.01074821, "D": 0.96752322}, "correct": false, "expected": "A", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.0003793, "B": 0.99780983, "C": 0.00132387, "D": 0.00048703}, "correct": false, "expected": "A", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.81198305, "B": 0.08558239, "C": 0.01685217, "D": 0.08558239}, "correct": false, "expected": "D", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.9985593, "B": 0.00055229, "C": 0.00070915, "D": 0.0001793}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00033494, "B": 0.99843019, "C": 0.00103168, "D": 0.00020315}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00279363, "B": 0.00042842, "C": 0.00217568, "D": 0.99460226}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.83312821, "B": 0.06035161, "C": 0.05326011, "D": 0.05326011}, "correct": false, "expected": "D", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00062376, "B": 0.00132049, "C": 0.00279548, "D": 0.9952603}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.38544163, "B": 0.34015104, "C": 0.04062525, "D": 0.23378217}, "correct": false, "expected": "B", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.02500182, "B": 0.93818629, "C": 0.01181002, "D": 0.02500182}, "correct": false, "expected": "D", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.06449899, "B": 0.8903811, "C": 0.03912061, "D": 0.00599934}, "correct": false, "expected": "A", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.998703, "B": 0.00080369, "C": 0.00033503, "D": 0.00015826}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.994605, "B": 0.00169443, "C": 0.00279364, "D": 0.00090696}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.9913556, "B": 0.00315527, "C": 0.00191377, "D": 0.00357539}, "correct": false, "expected": "C", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.99302411, "B": 0.00316058, "C": 0.0027892, "D": 0.00102609}, "correct": false, "expected": "B", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00090798, "B": 0.99571592, "C": 0.00246813, "D": 0.00090798}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00505846, "B": 0.0018609, "C": 0.02910941, "D": 0.96397126}, "correct": false, "expected": "C", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.71590847, "B": 0.09688769, "C": 0.00619382, "D": 0.18101002}, "correct": false, "expected": "D", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01541202, "B": 0.02541013, "C": 0.95350814, "D": 0.00566977}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01855717, "B": 0.8941347, "C": 0.08316748, "D": 0.00414066}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00187169, "B": 0.00576522, "C": 0.96956116, "D": 0.02280189}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.87721205, "B": 0.00316372, "C": 0.11871775, "D": 0.00090642}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00958618, "B": 0.977817, "C": 0.00513111, "D": 0.00746572}, "correct": false, "expected": "A", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00758572, "B": 0.04365281, "C": 0.07197131, "D": 0.87679011}, "correct": false, "expected": "C", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.84167266, "B": 0.08871165, "C": 0.03263519, "D": 0.03698052}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.95482135, "B": 0.02245525, "C": 0.00729015, "D": 0.01543325}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00013979, "B": 0.99962699, "C": 0.0001584, "D": 7.482e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.99445289, "B": 0.00217536, "C": 0.002465, "D": 0.00090682}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.99801862, "B": 0.00116856, "C": 0.00055199, "D": 0.00026074}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00242688, "B": 0.00453401, "C": 0.97907341, "D": 0.01396573}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.99687767, "B": 0.00080222, "C": 0.00218066, "D": 0.00013941}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01050472, "B": 0.04154696, "C": 0.94560438, "D": 0.00234392}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.15470326, "B": 0.53996742, "C": 0.28902376, "D": 0.0163056}, "correct": false, "expected": "C", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00048763, "B": 0.9990539, "C": 0.00033515, "D": 0.00012329}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.08998808, "B": 0.02921485, "C": 0.21587016, "D": 0.66492689}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.99964571, "B": 0.00012337, "C": 0.0002034, "D": 2.753e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00597899, "B": 0.78309357, "C": 0.05672711, "D": 0.15420027}, "correct": false, "expected": "C", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.16418515, "B": 0.83380097, "C": 0.00125357, "D": 0.00076033}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00037973, "B": 0.00037973, "C": 0.99894482, "D": 0.00029573}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00062497, "B": 0.99718958, "C": 0.00169883, "D": 0.00048672}, "correct": false, "expected": "D", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00530914, "B": 0.0941067, "C": 0.89285952, "D": 0.00772475}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.02231761, "B": 0.30808514, "C": 0.65221626, "D": 0.01738097}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.36923271, "B": 0.53723043, "C": 0.01114986, "D": 0.08238696}, "correct": false, "expected": "A", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.61528057, "B": 0.37318653, "C": 0.00469773, "D": 0.00683515}, "correct": false, "expected": "B", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.04826921, "B": 0.19090845, "C": 0.75505745, "D": 0.00576493}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.99609655, "B": 0.00149757, "C": 0.00149757, "D": 0.00090832}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01045581, "B": 0.03649436, "C": 0.01184798, "D": 0.94120181}, "correct": false, "expected": "B", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00356252, "B": 0.98778927, "C": 0.00587361, "D": 0.0027745}, "correct": false, "expected": "C", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00517862, "B": 0.00277192, "C": 0.00517862, "D": 0.98687077}, "correct": false, "expected": "A", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.89548635, "B": 0.0832932, "C": 0.01127251, "D": 0.00994796}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00062448, "B": 0.0010296, "C": 0.00192355, "D": 0.99642241}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00017932, "B": 0.99864906, "C": 0.00091065, "D": 0.00026091}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00015834, "B": 0.99923062, "C": 0.00048772, "D": 0.00012331}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00033313, "B": 0.99305665, "C": 0.00590493, "D": 0.00070524}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.99709523, "B": 0.0010303, "C": 0.00132293, "D": 0.00055148}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.99772865, "B": 0.00169975, "C": 0.00048699, "D": 8.463e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.07207194, "B": 0.87801594, "C": 0.02339833, "D": 0.02651379}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 9.606e-05, "B": 0.99946827, "C": 0.00029589, "D": 0.00013977}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.9424172, "B": 0.02845853, "C": 0.01186328, "D": 0.01726097}, "correct": false, "expected": "D", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00515939, "B": 0.00850639, "C": 0.00312933, "D": 0.9832049}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.87490505, "B": 0.06337791, "C": 0.04355896, "D": 0.01815808}, "correct": false, "expected": "C", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00557572, "B": 0.00383213, "C": 0.05290093, "D": 0.93769133}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.40368173, "B": 0.27744612, "C": 0.31438765, "D": 0.0044845}, "correct": false, "expected": "B", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00192413, "B": 0.00055127, "C": 0.0008021, "D": 0.99672252}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00026112, "B": 0.00020336, "C": 0.99946946, "D": 6.602e-05}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00048692, "B": 0.00149981, "C": 0.99758363, "D": 0.0004297}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.09181255, "B": 0.13358647, "C": 0.4114753, "D": 0.36312568}, "correct": false, "expected": "D", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00149937, "B": 0.99729604, "C": 0.00090942, "D": 0.00029524}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00725356, "B": 0.9500286, "C": 0.0253174, "D": 0.01740038}, "correct": false, "expected": "D", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.39235455, "B": 0.57087249, "C": 0.01723886, "D": 0.01953418}, "correct": false, "expected": "A", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01453557, "B": 0.3748773, "C": 0.54544324, "D": 0.06514391}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00245371, "B": 0.00519451, "C": 0.98989809, "D": 0.00245371}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00047021, "B": 0.14773461, "C": 0.85015398, "D": 0.00164118}, "correct": true, "expected": "C", "predicted": "C", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.90763736, "B": 0.0451886, "C": 0.0451886, "D": 0.00198545}, "correct": true, "expected": "A", "predicted": "A", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.21965301, "B": 0.76666433, "C": 0.00516574, "D": 0.00851687}, "correct": false, "expected": "A", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.0008027, "B": 0.99747759, "C": 0.00116793, "D": 0.00055169}, "correct": true, "expected": "B", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01498052, "B": 0.63698804, "C": 0.34095514, "D": 0.0070763}, "correct": false, "expected": "D", "predicted": "B", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.00278383, "B": 0.00090378, "C": 0.00520088, "D": 0.99111158}, "correct": true, "expected": "D", "predicted": "D", "subject": "econometrics"}
|
||||
{"choice_probabilities": {"A": 0.01684211, "B": 0.9195478, "C": 0.00482534, "D": 0.05878473}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.33201173, "B": 0.33201173, "C": 0.22818811, "D": 0.10778843}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01759614, "B": 0.51423466, "C": 0.11474127, "D": 0.35342798}, "correct": false, "expected": "D", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0045487, "B": 0.67508632, "C": 0.31888822, "D": 0.00147675}, "correct": false, "expected": "C", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0096666, "B": 0.98601961, "C": 0.00215691, "D": 0.00215691}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00436171, "B": 0.94186717, "C": 0.03652016, "D": 0.0172509}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 4.539e-05, "B": 5.143e-05, "C": 0.00017951, "D": 0.99972361}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.97359622, "B": 0.00309874, "C": 0.00309874, "D": 0.02020635}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 5.827e-05, "B": 6.603e-05, "C": 0.00023048, "D": 0.99964523}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99887854, "B": 0.00043026, "C": 0.00043026, "D": 0.00026097}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01242618, "B": 0.98713338, "C": 0.00037524, "D": 6.521e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.04489193, "B": 0.00368495, "C": 0.33170903, "D": 0.61971402}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.78069073, "B": 0.0388683, "C": 0.15372711, "D": 0.02671377}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.93984848, "B": 0.02838097, "C": 0.00338962, "D": 0.02838097}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99538887, "B": 0.00279584, "C": 0.00090768, "D": 0.00090768}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00405064, "B": 0.99115771, "C": 0.00458997, "D": 0.00020167}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00459877, "B": 0.00102612, "C": 0.00131757, "D": 0.99305749}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.07793976, "B": 0.83793139, "C": 0.0687816, "D": 0.01534725}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.02016018, "B": 0.97137171, "C": 0.00396977, "D": 0.00449834}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 1.67e-05, "B": 0.99989831, "C": 6.605e-05, "D": 1.892e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01404572, "B": 0.00048062, "C": 0.00079241, "D": 0.98468125}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.70188934, "B": 0.02119522, "C": 0.01870472, "D": 0.25821066}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00969752, "B": 0.98917401, "C": 0.00070249, "D": 0.00042608}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 7.473e-05, "B": 0.00023018, "C": 0.00132462, "D": 0.99837047}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00043043, "B": 0.99926442, "C": 0.00023039, "D": 7.48e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00961737, "B": 0.00189377, "C": 0.00749002, "D": 0.98099881}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00080244, "B": 0.9971469, "C": 0.00149915, "D": 0.00055151}, "correct": false, "expected": "D", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01793153, "B": 0.97902817, "C": 0.00242677, "D": 0.00061358}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.82614261, "B": 0.12669307, "C": 0.03203303, "D": 0.01513133}, "correct": false, "expected": "B", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0035754, "B": 0.99135876, "C": 0.0035754, "D": 0.00149045}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99929368, "B": 0.0002304, "C": 0.00037986, "D": 9.604e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00015838, "B": 9.607e-05, "C": 0.00023045, "D": 0.99951518}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 2.753e-05, "B": 3.12e-05, "C": 0.00012339, "D": 0.99981791}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00020219, "B": 0.00015747, "C": 0.00590894, "D": 0.99373144}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.9992398, "B": 0.00026106, "C": 0.00020331, "D": 0.00029582}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00080251, "B": 0.99723792, "C": 0.00169891, "D": 0.00026054}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.90943819, "B": 0.08459093, "C": 0.00225427, "D": 0.00371667}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00062579, "B": 0.00048737, "C": 0.99850732, "D": 0.00037956}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00660958, "B": 0.00660958, "C": 0.00583293, "D": 0.98094797}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01158359, "B": 0.02778758, "C": 0.92019814, "D": 0.04043068}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 8.478e-05, "B": 5.142e-05, "C": 0.00029592, "D": 0.99956781}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 3.535e-05, "B": 3.535e-05, "C": 9.61e-05, "D": 0.99983311}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01071721, "B": 0.11522088, "C": 0.02268833, "D": 0.85137349}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00045714, "B": 0.7294026, "C": 0.00180801, "D": 0.26833224}, "correct": false, "expected": "D", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00115962, "B": 0.53011364, "C": 0.46782362, "D": 0.00090311}, "correct": false, "expected": "C", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.31292415, "B": 0.31292415, "C": 0.1304462, "D": 0.24370557}, "correct": false, "expected": "C", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.78126389, "B": 0.1357633, "C": 0.04407589, "D": 0.03889684}, "correct": false, "expected": "B", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 4.539e-05, "B": 0.9998455, "C": 9.61e-05, "D": 1.301e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.45804876, "B": 0.314812, "C": 0.21636692, "D": 0.01077227}, "correct": false, "expected": "C", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00955938, "B": 0.97508323, "C": 0.00955938, "D": 0.00579806}, "correct": false, "expected": "D", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00026097, "B": 0.00012327, "C": 0.0007094, "D": 0.99890637}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.98762578, "B": 0.01097153, "C": 0.00070139, "D": 0.00070139}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.98487109, "B": 0.00516813, "C": 0.00244125, "D": 0.00751958}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00043041, "B": 0.99923182, "C": 0.00015834, "D": 0.00017942}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99879187, "B": 0.00080376, "C": 0.00029569, "D": 0.00010878}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99933642, "B": 0.00015836, "C": 0.00043046, "D": 7.48e-05}, "correct": false, "expected": "C", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00770679, "B": 0.61222625, "C": 0.00873294, "D": 0.37133399}, "correct": false, "expected": "D", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99128813, "B": 0.00589442, "C": 0.00090394, "D": 0.00191364}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00091074, "B": 0.99875021, "C": 0.00023027, "D": 0.00010877}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0109757, "B": 0.98800141, "C": 0.00079508, "D": 0.00022779}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00015833, "B": 0.00037981, "C": 0.0002958, "D": 0.99916601}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 5.828e-05, "B": 9.609e-05, "C": 0.99980026, "D": 4.539e-05}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01241177, "B": 0.00089911, "C": 0.00070022, "D": 0.98598897}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00855119, "B": 0.00115728, "C": 0.00190803, "D": 0.98838347}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00218083, "B": 0.00048661, "C": 0.00037897, "D": 0.99695361}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99012637, "B": 0.00357095, "C": 0.00315136, "D": 0.00315136}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00042977, "B": 0.00080291, "C": 0.99773633, "D": 0.00103096}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00091077, "B": 0.99877721, "C": 0.00020322, "D": 0.00010878}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00457371, "B": 0.00587276, "C": 0.98764694, "D": 0.00190661}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01098291, "B": 0.00033166, "C": 0.98865044, "D": 3.496e-05}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00062615, "B": 0.99907458, "C": 0.00020328, "D": 9.602e-05}, "correct": false, "expected": "A", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 4.537e-05, "B": 4.537e-05, "C": 0.00048781, "D": 0.99942148}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01582277, "B": 0.00214138, "C": 0.97892022, "D": 0.00311569}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99891818, "B": 0.00080386, "C": 0.00020325, "D": 7.477e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00042938, "B": 0.00080219, "C": 0.00192436, "D": 0.99684405}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00623036, "B": 0.01693588, "C": 0.05216617, "D": 0.92466754}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00013974, "B": 0.99926752, "C": 0.00055268, "D": 4.004e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99977881, "B": 0.00012338, "C": 8.48e-05, "D": 1.3e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.75162739, "B": 0.24401768, "C": 0.00271079, "D": 0.00164418}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00015839, "B": 0.999529, "C": 0.00026114, "D": 5.142e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.38132274, "B": 0.43209529, "C": 0.15895897, "D": 0.02762293}, "correct": false, "expected": "C", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01111254, "B": 0.77904934, "C": 0.05643415, "D": 0.15340391}, "correct": false, "expected": "C", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.90898836, "B": 0.00420945, "C": 0.08454908, "D": 0.00225316}, "correct": false, "expected": "C", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00090243, "B": 0.00755593, "C": 0.00191044, "D": 0.98963124}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01461591, "B": 0.07422572, "C": 0.00690407, "D": 0.90425432}, "correct": false, "expected": "B", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0330602, "B": 0.96616077, "C": 0.00060552, "D": 0.00017348}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.97910005, "B": 0.01793284, "C": 0.00166801, "D": 0.00129905}, "correct": false, "expected": "B", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99795145, "B": 0.00091001, "C": 0.00070872, "D": 0.00042986}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.02003411, "B": 0.01072349, "C": 0.00394495, "D": 0.96529752}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.35369885, "B": 0.51462877, "C": 0.08942907, "D": 0.0422433}, "correct": false, "expected": "C", "predicted": "B", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.74574596, "B": 0.00645197, "C": 0.24210826, "D": 0.00569384}, "correct": false, "expected": "C", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0008035, "B": 0.00043008, "C": 0.99847072, "D": 0.00029559}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00170022, "B": 0.0002301, "C": 0.99800378, "D": 6.592e-05}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.94677871, "B": 0.0367106, "C": 0.00722874, "D": 0.00928189}, "correct": false, "expected": "B", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99798691, "B": 0.00103122, "C": 0.00055197, "D": 0.00042988}, "correct": false, "expected": "B", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.86690331, "B": 0.06279827, "C": 0.06279827, "D": 0.00750018}, "correct": false, "expected": "C", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 7.481e-05, "B": 7.481e-05, "C": 0.00043049, "D": 0.99941993}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.9919228, "B": 0.00459351, "C": 0.00102495, "D": 0.00245873}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00043012, "B": 0.00080358, "C": 0.99856323, "D": 0.00020318}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00017855, "B": 0.00020233, "C": 0.00521814, "D": 0.99440092}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_macroeconomics"}
|
||||
{"choice_probabilities": {"A": 0.5082528, "B": 0.44853151, "C": 0.00639796, "D": 0.03681771}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.83260727, "B": 0.01345784, "C": 0.00924943, "D": 0.14468545}, "correct": false, "expected": "C", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 2.753e-05, "B": 0.99984229, "C": 8.48e-05, "D": 4.539e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 8.479e-05, "B": 8.479e-05, "C": 0.00010888, "D": 0.99972147}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00020311, "B": 0.00023015, "C": 0.00132445, "D": 0.99824226}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.92927611, "B": 0.02185448, "C": 0.00803981, "D": 0.04082954}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01783663, "B": 0.05494069, "C": 0.40595984, "D": 0.52126276}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00131799, "B": 0.00070547, "C": 0.99337631, "D": 0.00460025}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00023023, "B": 0.00070916, "C": 0.0004874, "D": 0.99857318}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.54040968, "B": 0.37141779, "C": 0.08287451, "D": 0.00529799}, "correct": false, "expected": "B", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.05605473, "B": 0.00358346, "C": 0.87684345, "D": 0.06351833}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.84092206, "B": 0.14613034, "C": 0.00236198, "D": 0.01058565}, "correct": false, "expected": "C", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.000549, "B": 0.00278803, "C": 0.00405656, "D": 0.9926064}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0002031, "B": 0.00029551, "C": 0.99817705, "D": 0.00132436}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00013977, "B": 0.9995209, "C": 0.00023045, "D": 0.00010886}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0024267, "B": 0.00274981, "C": 0.01582405, "D": 0.9789995}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99469769, "B": 0.00358744, "C": 0.00055015, "D": 0.00116467}, "correct": false, "expected": "C", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00526954, "B": 0.36942324, "C": 0.01623132, "D": 0.60907596}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00020323, "B": 0.99881423, "C": 0.00055243, "D": 0.00043023}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00679844, "B": 0.00872937, "C": 0.78579259, "D": 0.19867951}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00192283, "B": 0.00132154, "C": 0.00070737, "D": 0.99604827}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.53747797, "B": 0.25388664, "C": 0.07273974, "D": 0.13589573}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00116327, "B": 0.00217328, "C": 0.0031621, "D": 0.99350137}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00023047, "B": 0.99961984, "C": 7.482e-05, "D": 7.482e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00037983, "B": 0.99921346, "C": 0.00020331, "D": 0.00020331}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00037902, "B": 0.99707127, "C": 0.00062489, "D": 0.0019248}, "correct": false, "expected": "D", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 7.481e-05, "B": 0.99949133, "C": 0.00023044, "D": 0.00020336}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00246863, "B": 0.00070727, "C": 0.00090816, "D": 0.99591595}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.11163976, "B": 0.44154376, "C": 0.30346829, "D": 0.14334826}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.02295402, "B": 0.00053983, "C": 0.97602975, "D": 0.0004764}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99487513, "B": 0.00192056, "C": 0.00217628, "D": 0.001028}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00192049, "B": 0.99483901, "C": 0.00192049, "D": 0.00131993}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.04393064, "B": 0.41680229, "C": 0.25280336, "D": 0.28646374}, "correct": false, "expected": "D", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99923146, "B": 0.00023038, "C": 0.00015834, "D": 0.00037984}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 7.481e-05, "B": 0.99950564, "C": 0.00015838, "D": 0.00026113}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99848205, "B": 0.00080351, "C": 0.00033495, "D": 0.00037955}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 9.609e-05, "B": 0.99978369, "C": 7.484e-05, "D": 4.539e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.03959822, "B": 0.05761507, "C": 0.00153539, "D": 0.90125126}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99270952, "B": 0.00191638, "C": 0.00131711, "D": 0.00405698}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99062514, "B": 0.00315294, "C": 0.00519832, "D": 0.00102361}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.7445997, "B": 0.00304301, "C": 0.01062114, "D": 0.24173613}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0024688, "B": 0.0003786, "C": 0.00116618, "D": 0.9959864}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00132452, "B": 0.00020312, "C": 0.00017925, "D": 0.9982931}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00116751, "B": 0.00080242, "C": 0.00090926, "D": 0.9971208}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00029566, "B": 0.99869943, "C": 0.00070925, "D": 0.00029566}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00012335, "B": 0.00020337, "C": 0.00013978, "D": 0.99953353}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0001584, "B": 8.479e-05, "C": 0.99963343, "D": 0.00012336}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00279469, "B": 0.99497789, "C": 0.0009073, "D": 0.00132012}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00020328, "B": 0.00023035, "C": 0.00048765, "D": 0.99907875}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 1.148e-05, "B": 0.99990952, "C": 5.144e-05, "D": 2.753e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0003796, "B": 0.0003796, "C": 0.99861491, "D": 0.00062586}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99825114, "B": 0.00103149, "C": 0.00023016, "D": 0.00048724}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00482419, "B": 0.43425971, "C": 0.00331561, "D": 0.5576005}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 7.484e-05, "B": 0.99979311, "C": 6.604e-05, "D": 6.604e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99926907, "B": 0.00048774, "C": 8.476e-05, "D": 0.00015835}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.96307933, "B": 0.03295476, "C": 0.00210673, "D": 0.00185918}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99948633, "B": 0.00029589, "C": 0.00010885, "D": 0.00010885}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.97798663, "B": 0.01231103, "C": 0.00658962, "D": 0.00311272}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00012337, "B": 0.99969578, "C": 9.608e-05, "D": 8.479e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00090965, "B": 0.99755549, "C": 0.00062519, "D": 0.00090965}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.02591933, "B": 0.9726156, "C": 0.00113882, "D": 0.00032628}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.05177633, "B": 0.00899738, "C": 0.38257822, "D": 0.55664802}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.03683434, "B": 0.94997019, "C": 0.00388231, "D": 0.00931318}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 7.482e-05, "B": 0.99960583, "C": 0.00026116, "D": 5.827e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.12141156, "B": 0.10714532, "C": 0.29125115, "D": 0.48019198}, "correct": false, "expected": "A", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.98567748, "B": 0.00402824, "C": 0.00276857, "D": 0.00752574}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.9997403, "B": 0.00010888, "C": 6.604e-05, "D": 8.48e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00012329, "B": 0.00055253, "C": 0.00033512, "D": 0.99898905}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00131934, "B": 0.00279304, "C": 0.00149501, "D": 0.99439257}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00015837, "B": 0.99940598, "C": 0.00029587, "D": 0.00013976}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.97875494, "B": 0.0158201, "C": 0.00089251, "D": 0.00453253}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 7.473e-05, "B": 0.00017928, "C": 0.00132469, "D": 0.99842131}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00059579, "B": 0.04732983, "C": 0.00142924, "D": 0.95064515}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 9.605e-05, "B": 0.99934465, "C": 0.00037988, "D": 0.00017944}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00062362, "B": 0.00316701, "C": 0.00116508, "D": 0.99504435}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00055043, "B": 0.0001787, "C": 0.00406717, "D": 0.99520373}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99564111, "B": 0.00149689, "C": 0.00169619, "D": 0.00116578}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00661234, "B": 0.00849041, "C": 0.98135787, "D": 0.00353933}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.0035897, "B": 0.00070685, "C": 0.00037835, "D": 0.99532509}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00116882, "B": 0.00029552, "C": 0.99824011, "D": 0.00029552}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.60761911, "B": 0.0076488, "C": 0.0161925, "D": 0.36853963}, "correct": false, "expected": "D", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00217416, "B": 0.99390781, "C": 0.00358459, "D": 0.00033342}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00043021, "B": 0.00037966, "C": 0.99875998, "D": 0.00043021}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00023039, "B": 0.00023039, "C": 0.00026107, "D": 0.99927813}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99860519, "B": 0.00080361, "C": 0.00029563, "D": 0.00029563}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00026115, "B": 0.99958926, "C": 7.482e-05, "D": 7.482e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99923742, "B": 0.00037984, "C": 0.00020331, "D": 0.00017942}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.01763551, "B": 0.96286625, "C": 0.00393501, "D": 0.01556328}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00029581, "B": 0.00037983, "C": 0.99920112, "D": 0.00012331}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00245553, "B": 0.0058905, "C": 0.99063045, "D": 0.00102362}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99975485, "B": 0.00010888, "C": 8.48e-05, "D": 5.143e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99928206, "B": 0.00020332, "C": 0.00017943, "D": 0.00033522}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.99943489, "B": 0.00020335, "C": 0.00015837, "D": 0.00020335}, "correct": true, "expected": "A", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00048743, "B": 0.00037961, "C": 0.00048743, "D": 0.99864548}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00192328, "B": 0.00029494, "C": 0.99628395, "D": 0.00149785}, "correct": true, "expected": "C", "predicted": "C", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.97645789, "B": 0.01788445, "C": 0.00213599, "D": 0.00352166}, "correct": false, "expected": "B", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.08493341, "B": 0.91312021, "C": 0.00073482, "D": 0.00121151}, "correct": true, "expected": "B", "predicted": "B", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00116841, "B": 0.00013955, "C": 0.00080303, "D": 0.99788898}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.69803059, "B": 0.29098243, "C": 0.00775442, "D": 0.00323252}, "correct": false, "expected": "B", "predicted": "A", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00132244, "B": 0.00062468, "C": 0.00132244, "D": 0.99673045}, "correct": true, "expected": "D", "predicted": "D", "subject": "high_school_microeconomics"}
|
||||
{"choice_probabilities": {"A": 0.00080335, "B": 0.99827945, "C": 0.00043, "D": 0.00048726}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.0021637, "B": 0.00115815, "C": 0.00755207, "D": 0.98912609}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00216322, "B": 0.00855571, "C": 0.98890519, "D": 0.00037591}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.11705892, "B": 0.00400554, "C": 0.01398069, "D": 0.86495483}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99573553, "B": 0.00359118, "C": 0.00037851, "D": 0.00029478}, "correct": false, "expected": "D", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.26834694, "B": 0.7294426, "C": 0.00096781, "D": 0.00124269}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00080123, "B": 0.00316894, "C": 0.00037848, "D": 0.99565142}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00132411, "B": 0.00026073, "C": 0.00042987, "D": 0.99798524}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00460902, "B": 0.99527127, "C": 7.45e-05, "D": 4.519e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00332021, "B": 0.92060286, "C": 0.07556769, "D": 0.00050917}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00751441, "B": 0.00585223, "C": 0.98419374, "D": 0.00243957}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00023049, "B": 0.99969828, "C": 4.005e-05, "D": 3.119e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.96942979, "B": 0.00068846, "C": 0.00060757, "D": 0.02927424}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.63184571, "B": 0.33820266, "C": 0.01311354, "D": 0.01683812}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99954081, "B": 0.00026114, "C": 0.00013978, "D": 5.827e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00316978, "B": 0.00042898, "C": 0.99591511, "D": 0.0004861}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.17061327, "B": 0.01586951, "C": 0.76463562, "D": 0.04888152}, "correct": false, "expected": "D", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.04030554, "B": 0.29781988, "C": 0.63048464, "D": 0.03138998}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.04010108, "B": 0.91269648, "C": 0.04544048, "D": 0.00176192}, "correct": false, "expected": "C", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00103066, "B": 0.00020295, "C": 0.00132339, "D": 0.99744296}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.26669785, "B": 0.00710726, "C": 0.00123506, "D": 0.72495991}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99801433, "B": 0.00080314, "C": 0.00080314, "D": 0.00037937}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00149962, "B": 0.00055168, "C": 0.00048686, "D": 0.9974618}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99100202, "B": 0.00667732, "C": 0.00116034, "D": 0.00116034}, "correct": false, "expected": "B", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.05808671, "B": 0.01664212, "C": 0.01664212, "D": 0.90862894}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00132453, "B": 0.00010872, "C": 0.00026082, "D": 0.99830592}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.98658979, "B": 0.01096002, "C": 0.00029207, "D": 0.00215816}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.8115176, "B": 0.18107405, "C": 0.00482546, "D": 0.00258288}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.11877213, "B": 0.87761384, "C": 0.00169419, "D": 0.00191977}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00080358, "B": 0.00013964, "C": 0.0004874, "D": 0.99856937}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.9972409, "B": 0.00070821, "C": 0.00055156, "D": 0.00149929}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.0027942, "B": 0.00070648, "C": 0.00169477, "D": 0.99480456}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00452902, "B": 0.00166614, "C": 0.01580785, "D": 0.977997}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 9.609e-05, "B": 0.9998104, "C": 6.604e-05, "D": 2.753e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00048775, "B": 0.00015835, "C": 0.99928802, "D": 6.601e-05}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00169719, "B": 0.99622691, "C": 0.00037869, "D": 0.00169719}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.93724251, "B": 0.00232319, "C": 0.05991592, "D": 0.00051837}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00192413, "B": 0.99672252, "C": 0.00055127, "D": 0.0008021}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.01404265, "B": 0.00069914, "C": 0.98446596, "D": 0.00079223}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00043051, "B": 0.99945718, "C": 8.477e-05, "D": 2.752e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00103209, "B": 0.99883181, "C": 8.472e-05, "D": 5.138e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99956137, "B": 0.00023046, "C": 8.478e-05, "D": 0.00012336}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.02904256, "B": 0.00734311, "C": 0.96175766, "D": 0.00185663}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00873837, "B": 0.28937513, "C": 0.69417489, "D": 0.00771159}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.9971053, "B": 0.00033449, "C": 0.00037903, "D": 0.00218116}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00168252, "B": 0.0010205, "C": 0.98761475, "D": 0.00968223}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.10615373, "B": 0.88881427, "C": 0.0028289, "D": 0.00220315}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.01388669, "B": 0.00032658, "C": 0.01225496, "D": 0.97353178}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00062444, "B": 0.00022972, "C": 0.00279853, "D": 0.99634737}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00033526, "B": 0.99940991, "C": 5.141e-05, "D": 0.00020335}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00855941, "B": 0.98933321, "C": 0.0014874, "D": 0.00062004}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99856794, "B": 0.0011692, "C": 0.00013964, "D": 0.00012323}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00520169, "B": 0.99126655, "C": 0.00315498, "D": 0.00037681}, "correct": false, "expected": "C", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.01923957, "B": 0.00260379, "C": 0.63712704, "D": 0.34102955}, "correct": false, "expected": "D", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00070927, "B": 0.00017933, "C": 0.00037965, "D": 0.99873179}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.48640993, "B": 0.33430433, "C": 0.13935879, "D": 0.03992696}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00012336, "B": 6.603e-05, "C": 0.00020339, "D": 0.99960726}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00445675, "B": 0.24333023, "C": 0.00270315, "D": 0.74950987}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.16358852, "B": 0.83077109, "C": 0.00299623, "D": 0.00264416}, "correct": false, "expected": "C", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.03181399, "B": 0.12582679, "C": 0.02186542, "D": 0.8204937}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.01387326, "B": 0.01080451, "C": 0.00273181, "D": 0.97259045}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99964356, "B": 0.0001584, "C": 0.00013979, "D": 5.827e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00589331, "B": 0.00054816, "C": 0.0024567, "D": 0.99110174}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00037994, "B": 0.99950409, "C": 8.478e-05, "D": 3.119e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00010888, "B": 3.535e-05, "C": 9.609e-05, "D": 0.99975961}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00070912, "B": 0.00055226, "C": 0.00023022, "D": 0.99850839}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.14707346, "B": 0.8463493, "C": 0.00570266, "D": 0.00087453}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00033492, "B": 0.99837261, "C": 0.00103162, "D": 0.00026083}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.000911, "B": 0.99903226, "C": 3.532e-05, "D": 2.142e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00029585, "B": 0.00013975, "C": 0.00023041, "D": 0.99933404}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00055241, "B": 0.00010878, "C": 0.00055241, "D": 0.99878639}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00456952, "B": 0.00115536, "C": 0.98674124, "D": 0.00753386}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.30563077, "B": 0.3924377, "C": 0.03221325, "D": 0.26971823}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99976748, "B": 9.609e-05, "C": 2.753e-05, "D": 0.00010888}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00062627, "B": 0.99926406, "C": 5.141e-05, "D": 5.825e-05}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00079959, "B": 0.00521398, "C": 0.99360871, "D": 0.0003777}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00591007, "B": 9.553e-05, "C": 7.44e-05, "D": 0.99392009}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.0227172, "B": 0.00737519, "C": 0.00394766, "D": 0.96595985}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.43482697, "B": 0.55832887, "C": 0.00201365, "D": 0.00483049}, "correct": false, "expected": "A", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00080305, "B": 0.00080305, "C": 0.9979068, "D": 0.00048707}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00080294, "B": 0.00116827, "C": 0.99776804, "D": 0.00026068}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00026111, "B": 5.826e-05, "C": 0.00023043, "D": 0.99945015}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99967086, "B": 0.00012337, "C": 6.603e-05, "D": 0.0001398}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.02235764, "B": 0.27237183, "C": 0.12865934, "D": 0.57661116}, "correct": false, "expected": "B", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00116723, "B": 0.00062477, "C": 0.00132265, "D": 0.99688536}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00149334, "B": 0.00459982, "C": 0.99328434, "D": 0.00062252}, "correct": false, "expected": "B", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.92385757, "B": 0.00484796, "C": 0.05212048, "D": 0.01917405}, "correct": false, "expected": "C", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.01153844, "B": 0.00545037, "C": 0.91661203, "D": 0.06639916}, "correct": false, "expected": "D", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00048782, "B": 3.534e-05, "C": 0.99944156, "D": 3.534e-05}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00192577, "B": 0.00012311, "C": 0.00037921, "D": 0.99757189}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00246256, "B": 0.00048491, "C": 0.9934696, "D": 0.00358301}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00020336, "B": 6.602e-05, "C": 0.00026112, "D": 0.99946958}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.02294341, "B": 0.97557867, "C": 0.00078508, "D": 0.00069283}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.0774278, "B": 0.12765685, "C": 0.73461443, "D": 0.06030082}, "correct": false, "expected": "D", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00667551, "B": 0.00168783, "C": 0.99073327, "D": 0.00090343}, "correct": true, "expected": "C", "predicted": "C", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.01798185, "B": 0.98177582, "C": 0.00012116, "D": 0.00012116}, "correct": true, "expected": "B", "predicted": "B", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00521683, "B": 0.00020228, "C": 0.00042822, "D": 0.99415261}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99896002, "B": 0.00043029, "C": 0.00043029, "D": 0.00017937}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.99946326, "B": 0.00048783, "C": 2.143e-05, "D": 2.752e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.08867099, "B": 0.06905703, "C": 0.00098505, "D": 0.8412869}, "correct": true, "expected": "D", "predicted": "D", "subject": "management"}
|
||||
{"choice_probabilities": {"A": 0.00247085, "B": 0.99681288, "C": 0.00022983, "D": 0.00048654}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.02888079, "B": 0.00644418, "C": 0.00827449, "D": 0.95640051}, "correct": false, "expected": "A", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.15032126, "B": 0.40861556, "C": 0.08046117, "D": 0.36060196}, "correct": false, "expected": "C", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.01138392, "B": 0.06550997, "C": 0.90433717, "D": 0.01876892}, "correct": false, "expected": "B", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.9930768, "B": 0.00521119, "C": 0.00054926, "D": 0.00116277}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.0011658, "B": 0.00246799, "C": 0.00070709, "D": 0.99565923}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99380612, "B": 0.00460224, "C": 0.00042807, "D": 0.00116363}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.8711164, "B": 0.08102644, "C": 0.03377685, "D": 0.01408029}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00131963, "B": 0.00048546, "C": 0.99460781, "D": 0.00358712}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.94016528, "B": 0.01044429, "C": 0.03217069, "D": 0.01721973}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00217937, "B": 0.0009085, "C": 0.9962877, "D": 0.0006244}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00991563, "B": 0.69513965, "C": 0.25572759, "D": 0.0392171}, "correct": false, "expected": "C", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.96784937, "B": 0.01564383, "C": 0.00575504, "D": 0.01075184}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00191004, "B": 0.98942751, "C": 0.00588335, "D": 0.0027791}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00403774, "B": 0.0027751, "C": 0.98800266, "D": 0.00518456}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00042978, "B": 0.00062533, "C": 0.99777669, "D": 0.00116828}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00358599, "B": 0.00149486, "C": 0.00062315, "D": 0.99429601}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.02893949, "B": 0.14696686, "C": 0.51296473, "D": 0.31112882}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.13801003, "B": 0.29216722, "C": 0.5458402, "D": 0.02398255}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.02523457, "B": 0.94692034, "C": 0.00819247, "D": 0.01965271}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00168395, "B": 0.00131146, "C": 0.98845273, "D": 0.00855179}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99544275, "B": 0.00169586, "C": 0.00169586, "D": 0.00116554}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00316042, "B": 0.00217213, "C": 0.00169165, "D": 0.99297583}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00070762, "B": 0.99640512, "C": 0.00070762, "D": 0.00217963}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.01220913, "B": 0.96989113, "C": 0.00950848, "D": 0.0083912}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00132077, "B": 0.99547297, "C": 0.00217759, "D": 0.00102862}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.03666751, "B": 0.00637186, "C": 0.64994729, "D": 0.30701336}, "correct": false, "expected": "A", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.01031924, "B": 0.03601769, "C": 0.92890847, "D": 0.02475457}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00218124, "B": 0.99714452, "C": 0.0002952, "D": 0.00037904}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99606329, "B": 0.00149752, "C": 0.00026023, "D": 0.00217888}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.01785077, "B": 0.97461909, "C": 0.00511433, "D": 0.00241584}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99578166, "B": 0.0024683, "C": 0.00042893, "D": 0.00132118}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.94879365, "B": 0.01054015, "C": 0.0038775, "D": 0.03678872}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00651726, "B": 0.00348844, "C": 0.02274747, "D": 0.96724689}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.98208058, "B": 0.01400863, "C": 0.0014765, "D": 0.00243433}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.98724222, "B": 0.00244713, "C": 0.00753769, "D": 0.00277296}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00581468, "B": 0.97787923, "C": 0.00399637, "D": 0.01230968}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00029518, "B": 0.99707186, "C": 0.00070809, "D": 0.0019248}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.06644171, "B": 0.91719943, "C": 0.00481302, "D": 0.01154584}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00042956, "B": 0.9972465, "C": 0.000625, "D": 0.00169893}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00516911, "B": 0.98505843, "C": 0.00313523, "D": 0.00663727}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.02778758, "B": 0.04043068, "C": 0.92019814, "D": 0.01158359}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00090842, "B": 0.99620491, "C": 0.00070748, "D": 0.00217919}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99199814, "B": 0.00405407, "C": 0.00116151, "D": 0.00278632}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99577916, "B": 0.00149709, "C": 0.00080134, "D": 0.00192231}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00102888, "B": 0.99572778, "C": 0.00132111, "D": 0.00192221}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00960537, "B": 0.00214325, "C": 0.97977465, "D": 0.00847671}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00048743, "B": 0.00015825, "C": 0.99864513, "D": 0.00070921}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.9596982, "B": 0.0199179, "C": 0.00830302, "D": 0.01208082}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.0214255, "B": 0.91103548, "C": 0.04002811, "D": 0.02751089}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99681658, "B": 0.00218053, "C": 0.0002951, "D": 0.00070791}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00037888, "B": 0.99672121, "C": 0.00042933, "D": 0.00247062}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.01398956, "B": 0.98074359, "C": 0.00312149, "D": 0.00214537}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.000803, "B": 0.00055189, "C": 0.000803, "D": 0.99784207}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.86536884, "B": 0.10335357, "C": 0.02613188, "D": 0.00514567}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.34299317, "B": 0.56550014, "C": 0.05960328, "D": 0.03190333}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.43806261, "B": 0.49638996, "C": 0.01322834, "D": 0.05231912}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00020322, "B": 0.00023027, "C": 0.99876273, "D": 0.00080374}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99736917, "B": 0.0011678, "C": 0.00029527, "D": 0.0011678}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99919969, "B": 0.0004304, "C": 0.00010882, "D": 0.00026105}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00090938, "B": 0.00080253, "C": 0.99725765, "D": 0.00103046}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99754059, "B": 0.001168, "C": 0.00026062, "D": 0.00103076}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99321896, "B": 0.00405906, "C": 0.00054933, "D": 0.00217266}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00043012, "B": 0.00037958, "C": 0.00062583, "D": 0.99856442}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99691963, "B": 0.00103011, "C": 0.00055138, "D": 0.00149881}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00664732, "B": 0.0040318, "C": 0.98654985, "D": 0.00277102}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.98732102, "B": 0.00587083, "C": 0.00277318, "D": 0.00403496}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99404484, "B": 0.00246399, "C": 0.00102714, "D": 0.00246399}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.11344764, "B": 0.06880946, "C": 0.73977149, "D": 0.07797135}, "correct": false, "expected": "D", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 8.48e-05, "B": 8.48e-05, "C": 8.48e-05, "D": 0.99974555}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99926132, "B": 0.00037985, "C": 0.00017943, "D": 0.00017943}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00037948, "B": 0.00070896, "C": 0.99828583, "D": 0.00062565}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.83631855, "B": 0.07778974, "C": 0.03242759, "D": 0.05346406}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.11632194, "B": 0.85950923, "C": 0.00842633, "D": 0.01574246}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.02853341, "B": 0.00926344, "C": 0.94489676, "D": 0.01730639}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00055247, "B": 0.00029572, "C": 0.00026097, "D": 0.99889094}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00033514, "B": 0.00029576, "C": 0.99903405, "D": 0.00033514}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.05505609, "B": 0.40681249, "C": 0.52235758, "D": 0.01577383}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00316916, "B": 0.00048601, "C": 0.00062404, "D": 0.99572092}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00244547, "B": 0.00853553, "C": 0.00244547, "D": 0.98657346}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.04185081, "B": 0.18756233, "C": 0.02876361, "D": 0.74182332}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00191737, "B": 0.00405908, "C": 0.00079928, "D": 0.99322432}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 8.449e-05, "B": 0.9960885, "C": 0.00279781, "D": 0.00102926}, "correct": false, "expected": "C", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.99274653, "B": 0.00217162, "C": 0.00048455, "D": 0.00459733}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00023038, "B": 0.00026105, "C": 0.00029581, "D": 0.99921274}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 9.602e-05, "B": 0.99909031, "C": 0.00055258, "D": 0.00026102}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00589858, "B": 0.00062171, "C": 0.99198842, "D": 0.00149139}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.06246097, "B": 0.52297932, "C": 0.05512162, "D": 0.35943809}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00070855, "B": 0.00055182, "C": 0.00103093, "D": 0.99770868}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.05212215, "B": 0.92388713, "C": 0.00905747, "D": 0.01493324}, "correct": false, "expected": "A", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.78234237, "B": 0.01115952, "C": 0.19780715, "D": 0.00869104}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00576674, "B": 0.01220819, "C": 0.96981692, "D": 0.01220819}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.96571398, "B": 0.02271142, "C": 0.00650693, "D": 0.0050676}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.98887759, "B": 0.006663, "C": 0.00131202, "D": 0.00314738}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00279698, "B": 0.00102895, "C": 0.00037853, "D": 0.99579549}, "correct": true, "expected": "D", "predicted": "D", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00033508, "B": 0.00017936, "C": 0.99885952, "D": 0.00062601}, "correct": true, "expected": "C", "predicted": "C", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.01359061, "B": 0.9527756, "C": 0.00824312, "D": 0.02539061}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.98101062, "B": 0.00749011, "C": 0.00400917, "D": 0.00749011}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.00043018, "B": 0.99870276, "C": 0.00037964, "D": 0.00048746}, "correct": true, "expected": "B", "predicted": "B", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.80964077, "B": 0.00424861, "C": 0.18065529, "D": 0.00545532}, "correct": true, "expected": "A", "predicted": "A", "subject": "marketing"}
|
||||
{"choice_probabilities": {"A": 0.98054081, "B": 0.00848334, "C": 0.00583051, "D": 0.00514541}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00503746, "B": 0.02558231, "C": 0.95996904, "D": 0.00941121}, "correct": false, "expected": "B", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.77339286, "B": 0.19554436, "C": 0.02646405, "D": 0.00459876}, "correct": false, "expected": "B", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.13290843, "B": 0.01090979, "C": 0.76483524, "D": 0.09134655}, "correct": false, "expected": "D", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.99483877, "B": 0.00246596, "C": 0.00246596, "D": 0.00022937}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.91117668, "B": 0.07479393, "C": 0.01147002, "D": 0.00255931}, "correct": false, "expected": "C", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.54575455, "B": 0.08369417, "C": 0.33101687, "D": 0.03953433}, "correct": false, "expected": "D", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00245379, "B": 0.00357025, "C": 0.98993039, "D": 0.00404562}, "correct": false, "expected": "D", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.0016806, "B": 0.98649025, "C": 0.00215794, "D": 0.00967121}, "correct": false, "expected": "D", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.95756811, "B": 0.01547765, "C": 0.02251984, "D": 0.00443442}, "correct": false, "expected": "C", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.9596982, "B": 0.01208082, "C": 0.0199179, "D": 0.00830302}, "correct": false, "expected": "D", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.08091662, "B": 0.46564299, "C": 0.32003143, "D": 0.13340895}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.95739949, "B": 0.02551383, "C": 0.01063575, "D": 0.00645091}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.78634393, "B": 0.17545705, "C": 0.03048987, "D": 0.00770904}, "correct": false, "expected": "B", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.12403356, "B": 0.18046775, "C": 0.43291965, "D": 0.26257905}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.04567825, "B": 0.03139417, "C": 0.00545549, "D": 0.91747212}, "correct": false, "expected": "B", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.11582171, "B": 0.51907688, "C": 0.21638343, "D": 0.14871801}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.07345122, "B": 0.29050523, "C": 0.61499953, "D": 0.02104413}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.01187121, "B": 0.73444605, "C": 0.23843972, "D": 0.01524294}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.12391837, "B": 0.18030016, "C": 0.5553636, "D": 0.14041792}, "correct": false, "expected": "B", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.01904744, "B": 0.80991751, "C": 0.01155285, "D": 0.15948221}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.39605936, "B": 0.50855029, "C": 0.07798871, "D": 0.01740163}, "correct": false, "expected": "D", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00148545, "B": 0.98803306, "C": 0.00968634, "D": 0.0007951}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.66222292, "B": 0.18973005, "C": 0.13039942, "D": 0.01764764}, "correct": false, "expected": "C", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.02147594, "B": 0.48879009, "C": 0.3806701, "D": 0.10906381}, "correct": false, "expected": "D", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.7081393, "B": 0.26050991, "C": 0.01665384, "D": 0.01469696}, "correct": false, "expected": "B", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00955438, "B": 0.00843171, "C": 0.00744096, "D": 0.9745729}, "correct": true, "expected": "D", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.21284594, "B": 0.51059085, "C": 0.18783589, "D": 0.08872738}, "correct": false, "expected": "D", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00278609, "B": 0.991916, "C": 0.00459348, "D": 0.00070443}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.04158704, "B": 0.23931688, "C": 0.6505307, "D": 0.06856544}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.22594544, "B": 0.42212164, "C": 0.17596647, "D": 0.17596647}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.10553495, "B": 0.88363326, "C": 0.00764493, "D": 0.00318688}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.81200755, "B": 0.07552847, "C": 0.04581033, "D": 0.06665364}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.22018161, "B": 0.76850933, "C": 0.00853737, "D": 0.00277168}, "correct": false, "expected": "A", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.12229288, "B": 0.07417438, "C": 0.37668857, "D": 0.42684409}, "correct": true, "expected": "D", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.34815517, "B": 0.57401085, "C": 0.0605003, "D": 0.01733363}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.12406696, "B": 0.03136904, "C": 0.80901819, "D": 0.03554578}, "correct": false, "expected": "B", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00406624, "B": 0.99497545, "C": 0.00080069, "D": 0.00015767}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.79636216, "B": 0.17769243, "C": 0.01458588, "D": 0.0113595}, "correct": false, "expected": "D", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.03624029, "B": 0.44149715, "C": 0.02198085, "D": 0.50028181}, "correct": true, "expected": "D", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.01787361, "B": 0.97586584, "C": 0.00351952, "D": 0.00274101}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00391923, "B": 0.14706817, "C": 0.00269365, "D": 0.8463189}, "correct": true, "expected": "D", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.90021139, "B": 0.07389385, "C": 0.02117094, "D": 0.00472388}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.28096667, "B": 0.19310538, "C": 0.06269214, "D": 0.46323574}, "correct": false, "expected": "A", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.99864835, "B": 0.00062588, "C": 0.00043016, "D": 0.00029564}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.80944616, "B": 0.18061185, "C": 0.00793553, "D": 0.00200642}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.09121276, "B": 0.40878722, "C": 0.09121276, "D": 0.40878722}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.7908659, "B": 0.15573072, "C": 0.0446176, "D": 0.00878573}, "correct": false, "expected": "B", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.0045949, "B": 0.00169037, "C": 0.99222296, "D": 0.00149175}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00213599, "B": 0.97645503, "C": 0.02026568, "D": 0.00114331}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.01224939, "B": 0.97308922, "C": 0.01224939, "D": 0.00241205}, "correct": false, "expected": "A", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.08996439, "B": 0.35581604, "C": 0.2771098, "D": 0.2771098}, "correct": false, "expected": "A", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.0203021, "B": 0.97820973, "C": 0.00101078, "D": 0.00047746}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.4450649, "B": 0.50432462, "C": 0.02215848, "D": 0.02845205}, "correct": false, "expected": "D", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00070886, "B": 0.00070886, "C": 0.99815232, "D": 0.00042995}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.02441026, "B": 0.05167652, "C": 0.00792485, "D": 0.91598833}, "correct": true, "expected": "D", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00048728, "B": 0.00062568, "C": 0.00055216, "D": 0.99833494}, "correct": true, "expected": "D", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.58920616, "B": 0.2783213, "C": 0.07037064, "D": 0.06210187}, "correct": false, "expected": "D", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.04373462, "B": 0.0636335, "C": 0.01419855, "D": 0.87843335}, "correct": true, "expected": "D", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00169094, "B": 0.00357972, "C": 0.00217121, "D": 0.99255812}, "correct": true, "expected": "D", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00017933, "B": 0.0009107, "C": 0.99870682, "D": 0.00020321}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.96387976, "B": 0.02568652, "C": 0.00649457, "D": 0.00393916}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.37460992, "B": 0.42448866, "C": 0.15616065, "D": 0.04474078}, "correct": false, "expected": "D", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.54858065, "B": 0.37703362, "C": 0.05781998, "D": 0.0165657}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.05807067, "B": 0.90837806, "C": 0.02743064, "D": 0.0061206}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.01589133, "B": 0.00079118, "C": 0.98316169, "D": 0.00015579}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.13514023, "B": 0.60565645, "C": 0.25247517, "D": 0.00672824}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.05089567, "B": 0.12209237, "C": 0.79614222, "D": 0.03086979}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.09057339, "B": 0.21727425, "C": 0.6692518, "D": 0.02290054}, "correct": false, "expected": "D", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.0376928, "B": 0.6681208, "C": 0.04839851, "D": 0.24578792}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.58052117, "B": 0.39898598, "C": 0.01547037, "D": 0.00502249}, "correct": false, "expected": "B", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.11507707, "B": 0.6622228, "C": 0.18973002, "D": 0.03297013}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.21936892, "B": 0.52623868, "C": 0.10362254, "D": 0.1507699}, "correct": false, "expected": "A", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.5896104, "B": 0.06214448, "C": 0.10245892, "D": 0.24578618}, "correct": false, "expected": "D", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.0070606, "B": 0.92475569, "C": 0.05911766, "D": 0.00906599}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.99841189, "B": 0.00116902, "C": 0.00026084, "D": 0.00015821}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.99638087, "B": 0.00279863, "C": 0.00048633, "D": 0.00033425}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.71285611, "B": 0.23143049, "C": 0.03132071, "D": 0.02439259}, "correct": false, "expected": "B", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.46522743, "B": 0.09160871, "C": 0.36231947, "D": 0.0808444}, "correct": false, "expected": "D", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00102878, "B": 0.00116576, "C": 0.99562752, "D": 0.00217793}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.03610958, "B": 0.93127835, "C": 0.01328397, "D": 0.01932807}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.86278862, "B": 0.11676575, "C": 0.00958472, "D": 0.01086091}, "correct": false, "expected": "B", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.99373454, "B": 0.00521464, "C": 0.0006228, "D": 0.00042804}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.49690223, "B": 0.30138645, "C": 0.14236487, "D": 0.05934651}, "correct": false, "expected": "B", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.01337713, "B": 0.03209008, "C": 0.12691872, "D": 0.82761401}, "correct": false, "expected": "C", "predicted": "D", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.01572971, "B": 0.02288659, "C": 0.85881311, "D": 0.1025706}, "correct": false, "expected": "D", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.3362709, "B": 0.38104486, "C": 0.2311154, "D": 0.05156882}, "correct": false, "expected": "A", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.4442299, "B": 0.39203152, "C": 0.12727401, "D": 0.03646461}, "correct": false, "expected": "C", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.30704236, "B": 0.11295457, "C": 0.57363063, "D": 0.00637246}, "correct": false, "expected": "A", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.14256482, "B": 0.34199521, "C": 0.43913054, "D": 0.07630945}, "correct": false, "expected": "B", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.80173898, "B": 0.09575406, "C": 0.05125346, "D": 0.05125346}, "correct": false, "expected": "D", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.80057812, "B": 0.01294014, "C": 0.17863312, "D": 0.00784859}, "correct": false, "expected": "D", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00601602, "B": 0.78794354, "C": 0.19922335, "D": 0.00681705}, "correct": false, "expected": "C", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.91113937, "B": 0.08474916, "C": 0.0025592, "D": 0.00155224}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.99938929, "B": 0.00033526, "C": 0.00017945, "D": 9.605e-05}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00521357, "B": 0.00102661, "C": 0.99353081, "D": 0.00022907}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.96387976, "B": 0.02568652, "C": 0.00649457, "D": 0.00393916}, "correct": true, "expected": "A", "predicted": "A", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.00017945, "B": 0.99941385, "C": 0.00020335, "D": 0.00020335}, "correct": true, "expected": "B", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.14321074, "B": 0.64182597, "C": 0.16227904, "D": 0.05268428}, "correct": false, "expected": "D", "predicted": "B", "subject": "professional_accounting"}
|
||||
{"choice_probabilities": {"A": 0.15865387, "B": 0.20371556, "C": 0.62748814, "D": 0.0101424}, "correct": true, "expected": "C", "predicted": "C", "subject": "professional_accounting"}
|
||||
100
evaluations/mmlu-finance-stratified-100/results.json
Normal file
100
evaluations/mmlu-finance-stratified-100/results.json
Normal file
@@ -0,0 +1,100 @@
|
||||
{
|
||||
"benchmark": "MMLU finance and business subset",
|
||||
"confidence_interval_95": [
|
||||
0.71513558,
|
||||
0.77929395
|
||||
],
|
||||
"dataset": "cais/mmlu",
|
||||
"dataset_revision": "c30699e8356da336a370243923dbaf21066bb9fe",
|
||||
"device": "mps",
|
||||
"dtype": "bfloat16",
|
||||
"evaluated_at": "2026-07-29T10:03:49.030340+00:00",
|
||||
"few_shot": 5,
|
||||
"macro_accuracy": 0.74857143,
|
||||
"micro_accuracy": 0.74857143,
|
||||
"model": ".artifacts/blum-transformers-v1/portable",
|
||||
"model_revision": "ad6f5cec7f729370d2976d8c78983521cb37ca83",
|
||||
"official_leaderboard_result": false,
|
||||
"sample_limit_per_subject": 100,
|
||||
"sample_size": 700,
|
||||
"status": "community_evaluation",
|
||||
"subjects": [
|
||||
{
|
||||
"accuracy": 0.77,
|
||||
"confidence_interval_95": [
|
||||
0.67845617,
|
||||
0.84156734
|
||||
],
|
||||
"correct": 77,
|
||||
"duration_seconds": 285.586,
|
||||
"sample_size": 100,
|
||||
"subject": "business_ethics"
|
||||
},
|
||||
{
|
||||
"accuracy": 0.65,
|
||||
"confidence_interval_95": [
|
||||
0.55254443,
|
||||
0.73635752
|
||||
],
|
||||
"correct": 65,
|
||||
"duration_seconds": 222.028,
|
||||
"sample_size": 100,
|
||||
"subject": "econometrics"
|
||||
},
|
||||
{
|
||||
"accuracy": 0.7,
|
||||
"confidence_interval_95": [
|
||||
0.60415145,
|
||||
0.78105115
|
||||
],
|
||||
"correct": 70,
|
||||
"duration_seconds": 187.56,
|
||||
"sample_size": 100,
|
||||
"subject": "high_school_macroeconomics"
|
||||
},
|
||||
{
|
||||
"accuracy": 0.87,
|
||||
"confidence_interval_95": [
|
||||
0.79019649,
|
||||
0.92242833
|
||||
],
|
||||
"correct": 87,
|
||||
"duration_seconds": 171.766,
|
||||
"sample_size": 100,
|
||||
"subject": "high_school_microeconomics"
|
||||
},
|
||||
{
|
||||
"accuracy": 0.87,
|
||||
"confidence_interval_95": [
|
||||
0.79019649,
|
||||
0.92242833
|
||||
],
|
||||
"correct": 87,
|
||||
"duration_seconds": 101.632,
|
||||
"sample_size": 100,
|
||||
"subject": "management"
|
||||
},
|
||||
{
|
||||
"accuracy": 0.92,
|
||||
"confidence_interval_95": [
|
||||
0.85001892,
|
||||
0.95890654
|
||||
],
|
||||
"correct": 92,
|
||||
"duration_seconds": 194.458,
|
||||
"sample_size": 100,
|
||||
"subject": "marketing"
|
||||
},
|
||||
{
|
||||
"accuracy": 0.46,
|
||||
"confidence_interval_95": [
|
||||
0.36560808,
|
||||
0.5573514
|
||||
],
|
||||
"correct": 46,
|
||||
"duration_seconds": 543.284,
|
||||
"sample_size": 100,
|
||||
"subject": "professional_accounting"
|
||||
}
|
||||
]
|
||||
}
|
||||
13
generation_config.json
Normal file
13
generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 151643,
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"temperature": 0.6,
|
||||
"top_k": 20,
|
||||
"top_p": 0.95,
|
||||
"transformers_version": "4.51.0"
|
||||
}
|
||||
151388
merges.txt
Normal file
151388
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model-00001-of-00010.safetensors
Normal file
3
model-00001-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:aaac72384c3bfb1c18e667f8bcf4659adfac097b4bdb9d96c61f613457c31a06
|
||||
size 877532632
|
||||
3
model-00002-of-00010.safetensors
Normal file
3
model-00002-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d9af1b28f14e6f142a8e7154e440b30976bf42031f35f9e1b40b571a0e895d7f
|
||||
size 883479608
|
||||
3
model-00003-of-00010.safetensors
Normal file
3
model-00003-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d61a9465b9e5f47175a4190d28366a211031465786441ee0ef663e2450a6e086
|
||||
size 883468408
|
||||
3
model-00004-of-00010.safetensors
Normal file
3
model-00004-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:defeb7d151f6fc5a385d282f7ca9cfbd706e0593b5d70cca673bd2b1e60235a5
|
||||
size 883479576
|
||||
3
model-00005-of-00010.safetensors
Normal file
3
model-00005-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2a4c51936a72b67db23f90807370eb17fa5856960f90c948c99c01f88d03bb61
|
||||
size 883489328
|
||||
3
model-00006-of-00010.safetensors
Normal file
3
model-00006-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fbfbbe4e9a6a3f380c8bdb8105f11fc79b5c7bff04fa6a64ca11186ee97b5fca
|
||||
size 857259032
|
||||
3
model-00007-of-00010.safetensors
Normal file
3
model-00007-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:16098efda2163bdb55066765d668a34635d0354f227fc395d86742d966b63814
|
||||
size 857259032
|
||||
3
model-00008-of-00010.safetensors
Normal file
3
model-00008-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:df3871a14de8cdcd6d4b0199dea17cc1529a5e788ceae2ebb896d260066700d8
|
||||
size 883479624
|
||||
3
model-00009-of-00010.safetensors
Normal file
3
model-00009-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9c5c3be2ffc5821987aef511f5c6317c6202a834c6de7b63089bc7d359707482
|
||||
size 888717024
|
||||
3
model-00010-of-00010.safetensors
Normal file
3
model-00010-of-00010.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:07ffc4a9032ec6f2ea40a346c90373bad445b6144a4f576b1e547e3ffff8b58f
|
||||
size 146817272
|
||||
405
model.safetensors.index.json
Normal file
405
model.safetensors.index.json
Normal file
@@ -0,0 +1,405 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_size": 8044936192
|
||||
},
|
||||
"weight_map": {
|
||||
"model.embed_tokens.weight": "model-00001-of-00010.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00010.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00010.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00010.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.0.self_attn.k_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.0.self_attn.q_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.self_attn.k_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.self_attn.q_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.self_attn.k_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.self_attn.q_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.self_attn.k_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.self_attn.q_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.self_attn.k_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.self_attn.q_norm.weight": "model-00002-of-00010.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.self_attn.k_norm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.self_attn.q_norm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.self_attn.k_norm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.self_attn.q_norm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.15.self_attn.k_norm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.15.self_attn.q_norm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.self_attn.k_norm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.self_attn.q_norm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.self_attn.k_norm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.self_attn.q_norm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.18.self_attn.k_norm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.18.self_attn.q_norm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.self_attn.k_norm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.self_attn.q_norm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.self_attn.k_norm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.self_attn.q_norm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.self_attn.k_norm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.self_attn.q_norm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.self_attn.k_norm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.self_attn.q_norm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00006-of-00010.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.22.self_attn.k_norm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.22.self_attn.q_norm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.self_attn.k_norm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.self_attn.q_norm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.self_attn.k_norm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.self_attn.q_norm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.self_attn.k_norm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.self_attn.q_norm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00007-of-00010.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.26.self_attn.k_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.26.self_attn.q_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.self_attn.k_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.self_attn.q_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.self_attn.k_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.self_attn.q_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.self_attn.k_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.self_attn.q_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00003-of-00010.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.3.self_attn.k_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.3.self_attn.q_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.30.self_attn.k_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.30.self_attn.q_norm.weight": "model-00008-of-00010.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.self_attn.k_norm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.self_attn.q_norm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.input_layernorm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.mlp.down_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.mlp.gate_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.mlp.up_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.post_attention_layernorm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.self_attn.k_norm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.self_attn.k_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.self_attn.o_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.self_attn.q_norm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.self_attn.q_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.32.self_attn.v_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.input_layernorm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.mlp.down_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.mlp.gate_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.mlp.up_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.post_attention_layernorm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.self_attn.k_norm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.self_attn.k_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.self_attn.o_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.self_attn.q_norm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.self_attn.q_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.33.self_attn.v_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.input_layernorm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.mlp.down_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.mlp.gate_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.mlp.up_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.post_attention_layernorm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.self_attn.k_norm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.self_attn.k_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.self_attn.o_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.self_attn.q_norm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.self_attn.q_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.34.self_attn.v_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.35.input_layernorm.weight": "model-00010-of-00010.safetensors",
|
||||
"model.layers.35.mlp.down_proj.weight": "model-00010-of-00010.safetensors",
|
||||
"model.layers.35.mlp.gate_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.35.mlp.up_proj.weight": "model-00010-of-00010.safetensors",
|
||||
"model.layers.35.post_attention_layernorm.weight": "model-00010-of-00010.safetensors",
|
||||
"model.layers.35.self_attn.k_norm.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.35.self_attn.k_proj.weight": "model-00009-of-00010.safetensors",
|
||||
"model.layers.35.self_attn.o_proj.weight": "model-00010-of-00010.safetensors",
|
||||
"model.layers.35.self_attn.q_norm.weight": "model-00010-of-00010.safetensors",
|
||||
"model.layers.35.self_attn.q_proj.weight": "model-00010-of-00010.safetensors",
|
||||
"model.layers.35.self_attn.v_proj.weight": "model-00010-of-00010.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.self_attn.k_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.self_attn.q_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.self_attn.k_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.self_attn.q_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.self_attn.k_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.self_attn.q_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.self_attn.k_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.self_attn.q_norm.weight": "model-00004-of-00010.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.self_attn.k_norm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.self_attn.q_norm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.self_attn.k_norm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.self_attn.q_norm.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00005-of-00010.safetensors",
|
||||
"model.norm.weight": "model-00010-of-00010.safetensors"
|
||||
}
|
||||
}
|
||||
40
pyproject.toml
Normal file
40
pyproject.toml
Normal file
@@ -0,0 +1,40 @@
|
||||
[build-system]
|
||||
requires = ["hatchling>=1.27"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "blum-finance"
|
||||
version = "0.2.0"
|
||||
description = "Structured local inference and opt-in contribution tools for BLUM Finance."
|
||||
requires-python = ">=3.11"
|
||||
license = { text = "Apache-2.0" }
|
||||
dependencies = [
|
||||
"pydantic>=2.10,<3",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
inference = [
|
||||
"accelerate>=1.2",
|
||||
"torch>=2.4",
|
||||
"transformers>=4.51",
|
||||
]
|
||||
mlx = [
|
||||
"mlx-lm>=0.31,<0.32",
|
||||
]
|
||||
hub = [
|
||||
"huggingface-hub>=0.30",
|
||||
]
|
||||
release = [
|
||||
"huggingface-hub>=0.30",
|
||||
"jinja2>=3.1",
|
||||
]
|
||||
test = [
|
||||
"pytest>=8.3",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
blum-contribute = "blum_finance.contributions:main"
|
||||
blum-memory-add = "blum_finance.memory:main"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["blum_finance"]
|
||||
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
Binary file not shown.
239
tokenizer_config.json
Normal file
239
tokenizer_config.json
Normal file
@@ -0,0 +1,239 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151646": {
|
||||
"content": "<|object_ref_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151647": {
|
||||
"content": "<|object_ref_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151648": {
|
||||
"content": "<|box_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151649": {
|
||||
"content": "<|box_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151650": {
|
||||
"content": "<|quad_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151651": {
|
||||
"content": "<|quad_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151652": {
|
||||
"content": "<|vision_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151653": {
|
||||
"content": "<|vision_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151654": {
|
||||
"content": "<|vision_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151655": {
|
||||
"content": "<|image_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151656": {
|
||||
"content": "<|video_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151657": {
|
||||
"content": "<tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151658": {
|
||||
"content": "</tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151659": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151660": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151661": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151662": {
|
||||
"content": "<|fim_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151663": {
|
||||
"content": "<|repo_name|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151664": {
|
||||
"content": "<|file_sep|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151665": {
|
||||
"content": "<tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151666": {
|
||||
"content": "</tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151667": {
|
||||
"content": "<think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151668": {
|
||||
"content": "</think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user